vendor update for E2E framework

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

vendor/k8s.io/kubernetes/test/e2e/framework/authorizer_util.go

@@ -0,0 +1,151 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"k8s.io/klog"
+	"sync"
+	"time"
+
+	authorizationv1beta1 "k8s.io/api/authorization/v1beta1"
+	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
+	v1beta1authorization "k8s.io/client-go/kubernetes/typed/authorization/v1beta1"
+	v1beta1rbac "k8s.io/client-go/kubernetes/typed/rbac/v1beta1"
+)
+
+const (
+	policyCachePollInterval = 100 * time.Millisecond
+	policyCachePollTimeout  = 5 * time.Second
+)
+
+// WaitForAuthorizationUpdate checks if the given user can perform the named verb and action.
+// If policyCachePollTimeout is reached without the expected condition matching, an error is returned
+func WaitForAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb string, resource schema.GroupResource, allowed bool) error {
+	return WaitForNamedAuthorizationUpdate(c, user, namespace, verb, "", resource, allowed)
+}
+
+// WaitForAuthorizationUpdate checks if the given user can perform the named verb and action on the named resource.
+// If policyCachePollTimeout is reached without the expected condition matching, an error is returned
+func WaitForNamedAuthorizationUpdate(c v1beta1authorization.SubjectAccessReviewsGetter, user, namespace, verb, resourceName string, resource schema.GroupResource, allowed bool) error {
+	review := &authorizationv1beta1.SubjectAccessReview{
+		Spec: authorizationv1beta1.SubjectAccessReviewSpec{
+			ResourceAttributes: &authorizationv1beta1.ResourceAttributes{
+				Group:     resource.Group,
+				Verb:      verb,
+				Resource:  resource.Resource,
+				Namespace: namespace,
+				Name:      resourceName,
+			},
+			User: user,
+		},
+	}
+	err := wait.Poll(policyCachePollInterval, policyCachePollTimeout, func() (bool, error) {
+		response, err := c.SubjectAccessReviews().Create(review)
+		// GKE doesn't enable the SAR endpoint.  Without this endpoint, we cannot determine if the policy engine
+		// has adjusted as expected.  In this case, simply wait one second and hope it's up to date
+		if apierrors.IsNotFound(err) {
+			klog.Info("SubjectAccessReview endpoint is missing")
+			time.Sleep(1 * time.Second)
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		if response.Status.Allowed != allowed {
+			return false, nil
+		}
+		return true, nil
+	})
+	return err
+}
+
+// BindClusterRole binds the cluster role at the cluster scope
+func BindClusterRole(c v1beta1rbac.ClusterRoleBindingsGetter, clusterRole, ns string, subjects ...rbacv1beta1.Subject) {
+	// Since the namespace names are unique, we can leave this lying around so we don't have to race any caches
+	_, err := c.ClusterRoleBindings().Create(&rbacv1beta1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns + "--" + clusterRole,
+		},
+		RoleRef: rbacv1beta1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     clusterRole,
+		},
+		Subjects: subjects,
+	})
+
+	// if we failed, don't fail the entire test because it may still work. RBAC may simply be disabled.
+	if err != nil {
+		klog.Errorf("Error binding clusterrole/%s for %q for %v\n", clusterRole, ns, subjects)
+	}
+}
+
+// BindClusterRoleInNamespace binds the cluster role at the namespace scope
+func BindClusterRoleInNamespace(c v1beta1rbac.RoleBindingsGetter, clusterRole, ns string, subjects ...rbacv1beta1.Subject) {
+	bindInNamespace(c, "ClusterRole", clusterRole, ns, subjects...)
+}
+
+// BindRoleInNamespace binds the role at the namespace scope
+func BindRoleInNamespace(c v1beta1rbac.RoleBindingsGetter, role, ns string, subjects ...rbacv1beta1.Subject) {
+	bindInNamespace(c, "Role", role, ns, subjects...)
+}
+
+func bindInNamespace(c v1beta1rbac.RoleBindingsGetter, roleType, role, ns string, subjects ...rbacv1beta1.Subject) {
+	// Since the namespace names are unique, we can leave this lying around so we don't have to race any caches
+	_, err := c.RoleBindings(ns).Create(&rbacv1beta1.RoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns + "--" + role,
+		},
+		RoleRef: rbacv1beta1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     roleType,
+			Name:     role,
+		},
+		Subjects: subjects,
+	})
+
+	// if we failed, don't fail the entire test because it may still work. RBAC may simply be disabled.
+	if err != nil {
+		klog.Errorf("Error binding %s/%s into %q for %v\n", roleType, role, ns, subjects)
+	}
+}
+
+var (
+	isRBACEnabledOnce sync.Once
+	isRBACEnabled     bool
+)
+
+func IsRBACEnabled(f *Framework) bool {
+	isRBACEnabledOnce.Do(func() {
+		crs, err := f.ClientSet.RbacV1().ClusterRoles().List(metav1.ListOptions{})
+		if err != nil {
+			Logf("Error listing ClusterRoles; assuming RBAC is disabled: %v", err)
+			isRBACEnabled = false
+		} else if crs == nil || len(crs.Items) == 0 {
+			Logf("No ClusterRoles found; assuming RBAC is disabled.")
+			isRBACEnabled = false
+		} else {
+			Logf("Found ClusterRoles; assuming RBAC is enabled.")
+			isRBACEnabled = true
+		}
+	})
+	return isRBACEnabled
+}

vendor/k8s.io/kubernetes/test/e2e/framework/cleanup.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import "sync"
+
+type CleanupActionHandle *int
+
+var cleanupActionsLock sync.Mutex
+var cleanupActions = map[CleanupActionHandle]func(){}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	cleanupActions[p] = fn
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	delete(cleanupActions, p)
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, fn := range cleanupActions {
+			list = append(list, fn)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/create.go

@@ -0,0 +1,583 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	rbac "k8s.io/api/rbac/v1"
+	storage "k8s.io/api/storage/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/test/e2e/framework/testfiles"
+)
+
+// LoadFromManifests loads .yaml or .json manifest files and returns
+// all items that it finds in them. It supports all items for which
+// there is a factory registered in Factories and .yaml files with
+// multiple items separated by "---". Files are accessed via the
+// "testfiles" package, which means they can come from a file system
+// or be built into the binary.
+//
+// LoadFromManifests has some limitations:
+// - aliases are not supported (i.e. use serviceAccountName instead of the deprecated serviceAccount,
+//   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.11/#podspec-v1-core)
+//   and silently ignored
+// - the latest stable API version for each item is used, regardless of what
+//   is specified in the manifest files
+func (f *Framework) LoadFromManifests(files ...string) ([]interface{}, error) {
+	var items []interface{}
+	err := visitManifests(func(data []byte) error {
+		// Ignore any additional fields for now, just determine what we have.
+		var what What
+		if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, &what); err != nil {
+			return errors.Wrap(err, "decode TypeMeta")
+		}
+
+		factory := Factories[what]
+		if factory == nil {
+			return errors.Errorf("item of type %+v not supported", what)
+		}
+
+		object := factory.New()
+		if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), data, object); err != nil {
+			return errors.Wrapf(err, "decode %+v", what)
+		}
+		items = append(items, object)
+		return nil
+	}, files...)
+
+	return items, err
+}
+
+func visitManifests(cb func([]byte) error, files ...string) error {
+	for _, fileName := range files {
+		data, err := testfiles.Read(fileName)
+		if err != nil {
+			Failf("reading manifest file: %v", err)
+		}
+
+		// Split at the "---" separator before working on
+		// individual item. Only works for .yaml.
+		//
+		// We need to split ourselves because we need access
+		// to each original chunk of data for
+		// runtime.DecodeInto. kubectl has its own
+		// infrastructure for this, but that is a lot of code
+		// with many dependencies.
+		items := bytes.Split(data, []byte("\n---"))
+
+		for _, item := range items {
+			if err := cb(item); err != nil {
+				return errors.Wrap(err, fileName)
+			}
+		}
+	}
+	return nil
+}
+
+// PatchItems modifies the given items in place such that each test
+// gets its own instances, to avoid conflicts between different tests
+// and between tests and normal deployments.
+//
+// This is done by:
+// - creating namespaced items inside the test's namespace
+// - changing the name of non-namespaced items like ClusterRole
+//
+// PatchItems has some limitations:
+// - only some common items are supported, unknown ones trigger an error
+// - only the latest stable API version for each item is supported
+func (f *Framework) PatchItems(items ...interface{}) error {
+	for _, item := range items {
+		// Uncomment when debugging the loading and patching of items.
+		// Logf("patching original content of %T:\n%s", item, PrettyPrint(item))
+		if err := f.patchItemRecursively(item); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CreateItems creates the items. Each of them must be an API object
+// of a type that is registered in Factory.
+//
+// It returns either a cleanup function or an error, but never both.
+//
+// Cleaning up after a test can be triggered in two ways:
+// - the test invokes the returned cleanup function,
+//   usually in an AfterEach
+// - the test suite terminates, potentially after
+//   skipping the test's AfterEach (https://github.com/onsi/ginkgo/issues/222)
+//
+// PatchItems has the some limitations as LoadFromManifests:
+// - only some common items are supported, unknown ones trigger an error
+// - only the latest stable API version for each item is supported
+func (f *Framework) CreateItems(items ...interface{}) (func(), error) {
+	var destructors []func() error
+	var cleanupHandle CleanupActionHandle
+	cleanup := func() {
+		if cleanupHandle == nil {
+			// Already done.
+			return
+		}
+		RemoveCleanupAction(cleanupHandle)
+
+		// TODO (?): use same logic as framework.go for determining
+		// whether we are expected to clean up? This would change the
+		// meaning of the -delete-namespace and -delete-namespace-on-failure
+		// command line flags, because they would also start to apply
+		// to non-namespaced items.
+		for _, destructor := range destructors {
+			if err := destructor(); err != nil && !apierrs.IsNotFound(err) {
+				Logf("deleting failed: %s", err)
+			}
+		}
+	}
+	cleanupHandle = AddCleanupAction(cleanup)
+
+	var result error
+	for _, item := range items {
+		// Each factory knows which item(s) it supports, so try each one.
+		done := false
+		description := DescribeItem(item)
+		// Uncomment this line to get a full dump of the entire item.
+		// description = fmt.Sprintf("%s:\n%s", description, PrettyPrint(item))
+		Logf("creating %s", description)
+		for _, factory := range Factories {
+			destructor, err := factory.Create(f, item)
+			if destructor != nil {
+				destructors = append(destructors, func() error {
+					Logf("deleting %s", description)
+					return destructor()
+				})
+			}
+			if err == nil {
+				done = true
+				break
+			} else if errors.Cause(err) != ItemNotSupported {
+				result = err
+				break
+			}
+		}
+		if result == nil && !done {
+			result = errors.Errorf("item of type %T not supported", item)
+			break
+		}
+	}
+
+	if result != nil {
+		cleanup()
+		return nil, result
+	}
+
+	return cleanup, nil
+}
+
+// CreateFromManifests is a combination of LoadFromManifests,
+// PatchItems, patching with an optional custom function,
+// and CreateItems.
+func (f *Framework) CreateFromManifests(patch func(item interface{}) error, files ...string) (func(), error) {
+	items, err := f.LoadFromManifests(files...)
+	if err != nil {
+		return nil, errors.Wrap(err, "CreateFromManifests")
+	}
+	if err := f.PatchItems(items...); err != nil {
+		return nil, err
+	}
+	if patch != nil {
+		for _, item := range items {
+			if err := patch(item); err != nil {
+				return nil, err
+			}
+		}
+	}
+	return f.CreateItems(items...)
+}
+
+// What is a subset of metav1.TypeMeta which (in contrast to
+// metav1.TypeMeta itself) satisfies the runtime.Object interface.
+type What struct {
+	Kind string `json:"kind"`
+}
+
+func (in *What) DeepCopy() *What {
+	return &What{Kind: in.Kind}
+}
+
+func (in *What) DeepCopyInto(out *What) {
+	out.Kind = in.Kind
+}
+
+func (in *What) DeepCopyObject() runtime.Object {
+	return &What{Kind: in.Kind}
+}
+
+func (in *What) GetObjectKind() schema.ObjectKind {
+	return nil
+}
+
+// ItemFactory provides support for creating one particular item.
+// The type gets exported because other packages might want to
+// extend the set of pre-defined factories.
+type ItemFactory interface {
+	// New returns a new empty item.
+	New() runtime.Object
+
+	// Create is responsible for creating the item. It returns an
+	// error or a cleanup function for the created item.
+	// If the item is of an unsupported type, it must return
+	// an error that has ItemNotSupported as cause.
+	Create(f *Framework, item interface{}) (func() error, error)
+}
+
+// DescribeItem always returns a string that describes the item,
+// usually by calling out to cache.MetaNamespaceKeyFunc which
+// concatenates namespace (if set) and name. If that fails, the entire
+// item gets converted to a string.
+func DescribeItem(item interface{}) string {
+	key, err := cache.MetaNamespaceKeyFunc(item)
+	if err == nil && key != "" {
+		return fmt.Sprintf("%T: %s", item, key)
+	}
+	return fmt.Sprintf("%T: %s", item, item)
+}
+
+// ItemNotSupported is the error that Create methods
+// must return or wrap when they don't support the given item.
+var ItemNotSupported = errors.New("not supported")
+
+var Factories = map[What]ItemFactory{
+	{"ClusterRole"}:        &clusterRoleFactory{},
+	{"ClusterRoleBinding"}: &clusterRoleBindingFactory{},
+	{"DaemonSet"}:          &daemonSetFactory{},
+	{"Role"}:               &roleFactory{},
+	{"RoleBinding"}:        &roleBindingFactory{},
+	{"Secret"}:             &secretFactory{},
+	{"Service"}:            &serviceFactory{},
+	{"ServiceAccount"}:     &serviceAccountFactory{},
+	{"StatefulSet"}:        &statefulSetFactory{},
+	{"StorageClass"}:       &storageClassFactory{},
+}
+
+// PatchName makes the name of some item unique by appending the
+// generated unique name.
+func (f *Framework) PatchName(item *string) {
+	if *item != "" {
+		*item = *item + "-" + f.UniqueName
+	}
+}
+
+// PatchNamespace moves the item into the test's namespace.  Not
+// all items can be namespaced. For those, the name also needs to be
+// patched.
+func (f *Framework) PatchNamespace(item *string) {
+	if f.Namespace != nil {
+		*item = f.Namespace.GetName()
+	}
+}
+
+func (f *Framework) patchItemRecursively(item interface{}) error {
+	switch item := item.(type) {
+	case *rbac.Subject:
+		f.PatchNamespace(&item.Namespace)
+	case *rbac.RoleRef:
+		// TODO: avoid hard-coding this special name. Perhaps add a Framework.PredefinedRoles
+		// which contains all role names that are defined cluster-wide before the test starts?
+		// All those names are excempt from renaming. That list could be populated by querying
+		// and get extended by tests.
+		if item.Name != "e2e-test-privileged-psp" {
+			f.PatchName(&item.Name)
+		}
+	case *rbac.ClusterRole:
+		f.PatchName(&item.Name)
+	case *rbac.Role:
+		f.PatchNamespace(&item.Namespace)
+		// Roles are namespaced, but because for RoleRef above we don't
+		// know whether the referenced role is a ClusterRole or Role
+		// and therefore always renames, we have to do the same here.
+		f.PatchName(&item.Name)
+	case *storage.StorageClass:
+		f.PatchName(&item.Name)
+	case *v1.ServiceAccount:
+		f.PatchNamespace(&item.ObjectMeta.Namespace)
+	case *v1.Secret:
+		f.PatchNamespace(&item.ObjectMeta.Namespace)
+	case *rbac.ClusterRoleBinding:
+		f.PatchName(&item.Name)
+		for i := range item.Subjects {
+			if err := f.patchItemRecursively(&item.Subjects[i]); err != nil {
+				return errors.Wrapf(err, "%T", f)
+			}
+		}
+		if err := f.patchItemRecursively(&item.RoleRef); err != nil {
+			return errors.Wrapf(err, "%T", f)
+		}
+	case *rbac.RoleBinding:
+		f.PatchNamespace(&item.Namespace)
+		for i := range item.Subjects {
+			if err := f.patchItemRecursively(&item.Subjects[i]); err != nil {
+				return errors.Wrapf(err, "%T", f)
+			}
+		}
+		if err := f.patchItemRecursively(&item.RoleRef); err != nil {
+			return errors.Wrapf(err, "%T", f)
+		}
+	case *v1.Service:
+		f.PatchNamespace(&item.ObjectMeta.Namespace)
+	case *apps.StatefulSet:
+		f.PatchNamespace(&item.ObjectMeta.Namespace)
+	case *apps.DaemonSet:
+		f.PatchNamespace(&item.ObjectMeta.Namespace)
+	default:
+		return errors.Errorf("missing support for patching item of type %T", item)
+	}
+	return nil
+}
+
+// The individual factories all follow the same template, but with
+// enough differences in types and functions that copy-and-paste
+// looked like the least dirty approach. Perhaps one day Go will have
+// generics.
+
+type serviceAccountFactory struct{}
+
+func (f *serviceAccountFactory) New() runtime.Object {
+	return &v1.ServiceAccount{}
+}
+
+func (*serviceAccountFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*v1.ServiceAccount)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+	client := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create ServiceAccount")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type clusterRoleFactory struct{}
+
+func (f *clusterRoleFactory) New() runtime.Object {
+	return &rbac.ClusterRole{}
+}
+
+func (*clusterRoleFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*rbac.ClusterRole)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	Logf("Define cluster role %v", item.GetName())
+	client := f.ClientSet.RbacV1().ClusterRoles()
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create ClusterRole")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type clusterRoleBindingFactory struct{}
+
+func (f *clusterRoleBindingFactory) New() runtime.Object {
+	return &rbac.ClusterRoleBinding{}
+}
+
+func (*clusterRoleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*rbac.ClusterRoleBinding)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.RbacV1().ClusterRoleBindings()
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create ClusterRoleBinding")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type roleFactory struct{}
+
+func (f *roleFactory) New() runtime.Object {
+	return &rbac.Role{}
+}
+
+func (*roleFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*rbac.Role)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.RbacV1().Roles(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create Role")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type roleBindingFactory struct{}
+
+func (f *roleBindingFactory) New() runtime.Object {
+	return &rbac.RoleBinding{}
+}
+
+func (*roleBindingFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*rbac.RoleBinding)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.RbacV1().RoleBindings(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create RoleBinding")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type serviceFactory struct{}
+
+func (f *serviceFactory) New() runtime.Object {
+	return &v1.Service{}
+}
+
+func (*serviceFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*v1.Service)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.CoreV1().Services(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create Service")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type statefulSetFactory struct{}
+
+func (f *statefulSetFactory) New() runtime.Object {
+	return &apps.StatefulSet{}
+}
+
+func (*statefulSetFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*apps.StatefulSet)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.AppsV1().StatefulSets(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create StatefulSet")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type daemonSetFactory struct{}
+
+func (f *daemonSetFactory) New() runtime.Object {
+	return &apps.DaemonSet{}
+}
+
+func (*daemonSetFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*apps.DaemonSet)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.AppsV1().DaemonSets(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create DaemonSet")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type storageClassFactory struct{}
+
+func (f *storageClassFactory) New() runtime.Object {
+	return &storage.StorageClass{}
+}
+
+func (*storageClassFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*storage.StorageClass)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.StorageV1().StorageClasses()
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create StorageClass")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+type secretFactory struct{}
+
+func (f *secretFactory) New() runtime.Object {
+	return &v1.Secret{}
+}
+
+func (*secretFactory) Create(f *Framework, i interface{}) (func() error, error) {
+	item, ok := i.(*v1.Secret)
+	if !ok {
+		return nil, ItemNotSupported
+	}
+
+	client := f.ClientSet.CoreV1().Secrets(f.Namespace.GetName())
+	if _, err := client.Create(item); err != nil {
+		return nil, errors.Wrap(err, "create Secret")
+	}
+	return func() error {
+		return client.Delete(item.GetName(), &metav1.DeleteOptions{})
+	}, nil
+}
+
+// PrettyPrint returns a human-readable representation of an item.
+func PrettyPrint(item interface{}) string {
+	data, err := json.MarshalIndent(item, "", "  ")
+	if err == nil {
+		return string(data)
+	}
+	return fmt.Sprintf("%+v", item)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/deployment_util.go

@@ -0,0 +1,309 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/watch"
+	clientset "k8s.io/client-go/kubernetes"
+	scaleclient "k8s.io/client-go/scale"
+	watchtools "k8s.io/client-go/tools/watch"
+	appsinternal "k8s.io/kubernetes/pkg/apis/apps"
+	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
+	testutils "k8s.io/kubernetes/test/utils"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+func UpdateDeploymentWithRetries(c clientset.Interface, namespace, name string, applyUpdate testutils.UpdateDeploymentFunc) (*apps.Deployment, error) {
+	return testutils.UpdateDeploymentWithRetries(c, namespace, name, applyUpdate, Logf, Poll, pollShortTimeout)
+}
+
+// Waits for the deployment to clean up old rcs.
+func WaitForDeploymentOldRSsNum(c clientset.Interface, ns, deploymentName string, desiredRSNum int) error {
+	var oldRSs []*apps.ReplicaSet
+	var d *apps.Deployment
+
+	pollErr := wait.PollImmediate(Poll, 5*time.Minute, func() (bool, error) {
+		deployment, err := c.AppsV1().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		d = deployment
+
+		_, oldRSs, err = deploymentutil.GetOldReplicaSets(deployment, c.AppsV1())
+		if err != nil {
+			return false, err
+		}
+		return len(oldRSs) == desiredRSNum, nil
+	})
+	if pollErr == wait.ErrWaitTimeout {
+		pollErr = fmt.Errorf("%d old replica sets were not cleaned up for deployment %q", len(oldRSs)-desiredRSNum, deploymentName)
+		logReplicaSetsOfDeployment(d, oldRSs, nil)
+	}
+	return pollErr
+}
+
+func logReplicaSetsOfDeployment(deployment *apps.Deployment, allOldRSs []*apps.ReplicaSet, newRS *apps.ReplicaSet) {
+	testutils.LogReplicaSetsOfDeployment(deployment, allOldRSs, newRS, Logf)
+}
+
+func WaitForObservedDeployment(c clientset.Interface, ns, deploymentName string, desiredGeneration int64) error {
+	return testutils.WaitForObservedDeployment(c, ns, deploymentName, desiredGeneration)
+}
+
+func WaitForDeploymentWithCondition(c clientset.Interface, ns, deploymentName, reason string, condType apps.DeploymentConditionType) error {
+	return testutils.WaitForDeploymentWithCondition(c, ns, deploymentName, reason, condType, Logf, Poll, pollLongTimeout)
+}
+
+// WaitForDeploymentRevisionAndImage waits for the deployment's and its new RS's revision and container image to match the given revision and image.
+// Note that deployment revision and its new RS revision should be updated shortly most of the time, but an overwhelmed RS controller
+// may result in taking longer to relabel a RS.
+func WaitForDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName string, revision, image string) error {
+	return testutils.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, revision, image, Logf, Poll, pollLongTimeout)
+}
+
+func NewDeployment(deploymentName string, replicas int32, podLabels map[string]string, imageName, image string, strategyType apps.DeploymentStrategyType) *apps.Deployment {
+	zero := int64(0)
+	return &apps.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   deploymentName,
+			Labels: podLabels,
+		},
+		Spec: apps.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{MatchLabels: podLabels},
+			Strategy: apps.DeploymentStrategy{
+				Type: strategyType,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: v1.PodSpec{
+					TerminationGracePeriodSeconds: &zero,
+					Containers: []v1.Container{
+						{
+							Name:  imageName,
+							Image: image,
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// Waits for the deployment to complete, and don't check if rolling update strategy is broken.
+// Rolling update strategy is used only during a rolling update, and can be violated in other situations,
+// such as shortly after a scaling event or the deployment is just created.
+func WaitForDeploymentComplete(c clientset.Interface, d *apps.Deployment) error {
+	return testutils.WaitForDeploymentComplete(c, d, Logf, Poll, pollLongTimeout)
+}
+
+// Waits for the deployment to complete, and check rolling update strategy isn't broken at any times.
+// Rolling update strategy should not be broken during a rolling update.
+func WaitForDeploymentCompleteAndCheckRolling(c clientset.Interface, d *apps.Deployment) error {
+	return testutils.WaitForDeploymentCompleteAndCheckRolling(c, d, Logf, Poll, pollLongTimeout)
+}
+
+// WaitForDeploymentUpdatedReplicasGTE waits for given deployment to be observed by the controller and has at least a number of updatedReplicas
+func WaitForDeploymentUpdatedReplicasGTE(c clientset.Interface, ns, deploymentName string, minUpdatedReplicas int32, desiredGeneration int64) error {
+	return testutils.WaitForDeploymentUpdatedReplicasGTE(c, ns, deploymentName, minUpdatedReplicas, desiredGeneration, Poll, pollLongTimeout)
+}
+
+// WaitForDeploymentRollbackCleared waits for given deployment either started rolling back or doesn't need to rollback.
+// Note that rollback should be cleared shortly, so we only wait for 1 minute here to fail early.
+func WaitForDeploymentRollbackCleared(c clientset.Interface, ns, deploymentName string) error {
+	return testutils.WaitForDeploymentRollbackCleared(c, ns, deploymentName, Poll, pollShortTimeout)
+}
+
+// WatchRecreateDeployment watches Recreate deployments and ensures no new pods will run at the same time with
+// old pods.
+func WatchRecreateDeployment(c clientset.Interface, d *apps.Deployment) error {
+	if d.Spec.Strategy.Type != apps.RecreateDeploymentStrategyType {
+		return fmt.Errorf("deployment %q does not use a Recreate strategy: %s", d.Name, d.Spec.Strategy.Type)
+	}
+
+	w, err := c.AppsV1().Deployments(d.Namespace).Watch(metav1.SingleObject(metav1.ObjectMeta{Name: d.Name, ResourceVersion: d.ResourceVersion}))
+	if err != nil {
+		return err
+	}
+
+	status := d.Status
+
+	condition := func(event watch.Event) (bool, error) {
+		d := event.Object.(*apps.Deployment)
+		status = d.Status
+
+		if d.Status.UpdatedReplicas > 0 && d.Status.Replicas != d.Status.UpdatedReplicas {
+			_, allOldRSs, err := deploymentutil.GetOldReplicaSets(d, c.AppsV1())
+			newRS, nerr := deploymentutil.GetNewReplicaSet(d, c.AppsV1())
+			if err == nil && nerr == nil {
+				Logf("%+v", d)
+				logReplicaSetsOfDeployment(d, allOldRSs, newRS)
+				logPodsOfDeployment(c, d, append(allOldRSs, newRS))
+			}
+			return false, fmt.Errorf("deployment %q is running new pods alongside old pods: %#v", d.Name, status)
+		}
+
+		return *(d.Spec.Replicas) == d.Status.Replicas &&
+			*(d.Spec.Replicas) == d.Status.UpdatedReplicas &&
+			d.Generation <= d.Status.ObservedGeneration, nil
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+	_, err = watchtools.UntilWithoutRetry(ctx, w, condition)
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("deployment %q never completed: %#v", d.Name, status)
+	}
+	return err
+}
+
+func ScaleDeployment(clientset clientset.Interface, scalesGetter scaleclient.ScalesGetter, ns, name string, size uint, wait bool) error {
+	return ScaleResource(clientset, scalesGetter, ns, name, size, wait, appsinternal.Kind("Deployment"), appsinternal.Resource("deployments"))
+}
+
+func RunDeployment(config testutils.DeploymentConfig) error {
+	By(fmt.Sprintf("creating deployment %s in namespace %s", config.Name, config.Namespace))
+	config.NodeDumpFunc = DumpNodeDebugInfo
+	config.ContainerDumpFunc = LogFailedContainers
+	return testutils.RunDeployment(config)
+}
+
+func logPodsOfDeployment(c clientset.Interface, deployment *apps.Deployment, rsList []*apps.ReplicaSet) {
+	testutils.LogPodsOfDeployment(c, deployment, rsList, Logf)
+}
+
+func WaitForDeploymentRevision(c clientset.Interface, d *apps.Deployment, targetRevision string) error {
+	err := wait.PollImmediate(Poll, pollLongTimeout, func() (bool, error) {
+		deployment, err := c.AppsV1().Deployments(d.Namespace).Get(d.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		revision := deployment.Annotations[deploymentutil.RevisionAnnotation]
+		return revision == targetRevision, nil
+	})
+	if err != nil {
+		return fmt.Errorf("error waiting for revision to become %q for deployment %q: %v", targetRevision, d.Name, err)
+	}
+	return nil
+}
+
+// CheckDeploymentRevisionAndImage checks if the input deployment's and its new replica set's revision and image are as expected.
+func CheckDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName, revision, image string) error {
+	return testutils.CheckDeploymentRevisionAndImage(c, ns, deploymentName, revision, image)
+}
+
+func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*apps.Deployment, error) {
+	deploymentSpec := MakeDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, false, command)
+	deployment, err := client.AppsV1().Deployments(namespace).Create(deploymentSpec)
+	if err != nil {
+		return nil, fmt.Errorf("deployment %q Create API error: %v", deploymentSpec.Name, err)
+	}
+	Logf("Waiting deployment %q to complete", deploymentSpec.Name)
+	err = WaitForDeploymentComplete(client, deployment)
+	if err != nil {
+		return nil, fmt.Errorf("deployment %q failed to complete: %v", deploymentSpec.Name, err)
+	}
+	return deployment, nil
+}
+
+// MakeDeployment creates a deployment definition based on the namespace. The deployment references the PVC's
+// name.  A slice of BASH commands can be supplied as args to be run by the pod
+func MakeDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *apps.Deployment {
+	if len(command) == 0 {
+		command = "trap exit TERM; while true; do sleep 1; done"
+	}
+	zero := int64(0)
+	deploymentName := "deployment-" + string(uuid.NewUUID())
+	deploymentSpec := &apps.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      deploymentName,
+			Namespace: namespace,
+		},
+		Spec: apps.DeploymentSpec{
+			Replicas: &replicas,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: podLabels,
+			},
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: v1.PodSpec{
+					TerminationGracePeriodSeconds: &zero,
+					Containers: []v1.Container{
+						{
+							Name:    "write-pod",
+							Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+							Command: []string{"/bin/sh"},
+							Args:    []string{"-c", command},
+							SecurityContext: &v1.SecurityContext{
+								Privileged: &isPrivileged,
+							},
+						},
+					},
+					RestartPolicy: v1.RestartPolicyAlways,
+				},
+			},
+		},
+	}
+	var volumeMounts = make([]v1.VolumeMount, len(pvclaims))
+	var volumes = make([]v1.Volume, len(pvclaims))
+	for index, pvclaim := range pvclaims {
+		volumename := fmt.Sprintf("volume%v", index+1)
+		volumeMounts[index] = v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename}
+		volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}}
+	}
+	deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
+	deploymentSpec.Spec.Template.Spec.Volumes = volumes
+	if nodeSelector != nil {
+		deploymentSpec.Spec.Template.Spec.NodeSelector = nodeSelector
+	}
+	return deploymentSpec
+}
+
+// GetPodsForDeployment gets pods for the given deployment
+func GetPodsForDeployment(client clientset.Interface, deployment *apps.Deployment) (*v1.PodList, error) {
+	replicaSet, err := deploymentutil.GetNewReplicaSet(deployment, client.AppsV1())
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get new replica set for deployment %q: %v", deployment.Name, err)
+	}
+	if replicaSet == nil {
+		return nil, fmt.Errorf("expected a new replica set for deployment %q, found none", deployment.Name)
+	}
+	podListFunc := func(namespace string, options metav1.ListOptions) (*v1.PodList, error) {
+		return client.CoreV1().Pods(namespace).List(options)
+	}
+	rsList := []*apps.ReplicaSet{replicaSet}
+	podList, err := deploymentutil.ListPods(deployment, rsList, podListFunc)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to list Pods of Deployment %q: %v", deployment.Name, err)
+	}
+	return podList, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/exec_util.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"io"
+	"net/url"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/remotecommand"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+
+	. "github.com/onsi/gomega"
+)
+
+// ExecOptions passed to ExecWithOptions
+type ExecOptions struct {
+	Command []string
+
+	Namespace     string
+	PodName       string
+	ContainerName string
+
+	Stdin         io.Reader
+	CaptureStdout bool
+	CaptureStderr bool
+	// If false, whitespace in std{err,out} will be removed.
+	PreserveWhitespace bool
+}
+
+// ExecWithOptions executes a command in the specified container,
+// returning stdout, stderr and error. `options` allowed for
+// additional parameters to be passed.
+func (f *Framework) ExecWithOptions(options ExecOptions) (string, string, error) {
+	Logf("ExecWithOptions %+v", options)
+
+	config, err := LoadConfig()
+	ExpectNoError(err, "failed to load restclient config")
+
+	const tty = false
+
+	req := f.ClientSet.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(options.PodName).
+		Namespace(options.Namespace).
+		SubResource("exec").
+		Param("container", options.ContainerName)
+	req.VersionedParams(&v1.PodExecOptions{
+		Container: options.ContainerName,
+		Command:   options.Command,
+		Stdin:     options.Stdin != nil,
+		Stdout:    options.CaptureStdout,
+		Stderr:    options.CaptureStderr,
+		TTY:       tty,
+	}, legacyscheme.ParameterCodec)
+
+	var stdout, stderr bytes.Buffer
+	err = execute("POST", req.URL(), config, options.Stdin, &stdout, &stderr, tty)
+
+	if options.PreserveWhitespace {
+		return stdout.String(), stderr.String(), err
+	}
+	return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err
+}
+
+// ExecCommandInContainerWithFullOutput executes a command in the
+// specified container and return stdout, stderr and error
+func (f *Framework) ExecCommandInContainerWithFullOutput(podName, containerName string, cmd ...string) (string, string, error) {
+	return f.ExecWithOptions(ExecOptions{
+		Command:       cmd,
+		Namespace:     f.Namespace.Name,
+		PodName:       podName,
+		ContainerName: containerName,
+
+		Stdin:              nil,
+		CaptureStdout:      true,
+		CaptureStderr:      true,
+		PreserveWhitespace: false,
+	})
+}
+
+// ExecCommandInContainer executes a command in the specified container.
+func (f *Framework) ExecCommandInContainer(podName, containerName string, cmd ...string) string {
+	stdout, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, containerName, cmd...)
+	Logf("Exec stderr: %q", stderr)
+	ExpectNoError(err,
+		"failed to execute command in pod %v, container %v: %v",
+		podName, containerName, err)
+	return stdout
+}
+
+func (f *Framework) ExecShellInContainer(podName, containerName string, cmd string) string {
+	return f.ExecCommandInContainer(podName, containerName, "/bin/sh", "-c", cmd)
+}
+
+func (f *Framework) ExecCommandInPod(podName string, cmd ...string) string {
+	pod, err := f.PodClient().Get(podName, metav1.GetOptions{})
+	ExpectNoError(err, "failed to get pod")
+	Expect(pod.Spec.Containers).NotTo(BeEmpty())
+	return f.ExecCommandInContainer(podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
+func (f *Framework) ExecCommandInPodWithFullOutput(podName string, cmd ...string) (string, string, error) {
+	pod, err := f.PodClient().Get(podName, metav1.GetOptions{})
+	ExpectNoError(err, "failed to get pod")
+	Expect(pod.Spec.Containers).NotTo(BeEmpty())
+	return f.ExecCommandInContainerWithFullOutput(podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
+func (f *Framework) ExecShellInPod(podName string, cmd string) string {
+	return f.ExecCommandInPod(podName, "/bin/sh", "-c", cmd)
+}
+
+func (f *Framework) ExecShellInPodWithFullOutput(podName string, cmd string) (string, string, error) {
+	return f.ExecCommandInPodWithFullOutput(podName, "/bin/sh", "-c", cmd)
+}
+
+func execute(method string, url *url.URL, config *restclient.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error {
+	exec, err := remotecommand.NewSPDYExecutor(config, method, url)
+	if err != nil {
+		return err
+	}
+	return exec.Stream(remotecommand.StreamOptions{
+		Stdin:  stdin,
+		Stdout: stdout,
+		Stderr: stderr,
+		Tty:    tty,
+	})
+}

vendor/k8s.io/kubernetes/test/e2e/framework/flake_reporting_util.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+)
+
+type FlakeReport struct {
+	lock       sync.RWMutex
+	Flakes     []string `json:"flakes"`
+	FlakeCount int      `json:"flakeCount"`
+}
+
+func NewFlakeReport() *FlakeReport {
+	return &FlakeReport{
+		Flakes: []string{},
+	}
+}
+
+func buildDescription(optionalDescription ...interface{}) string {
+	switch len(optionalDescription) {
+	case 0:
+		return ""
+	default:
+		return fmt.Sprintf(optionalDescription[0].(string), optionalDescription[1:]...)
+	}
+}
+
+// RecordFlakeIfError records the error (if non-nil) as a flake along with an optional description.
+// This can be used as a replacement of framework.ExpectNoError() for non-critical errors that can
+// be considered as 'flakes' to avoid causing failures in tests.
+func (f *FlakeReport) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
+	if err == nil {
+		return
+	}
+	msg := fmt.Sprintf("Unexpected error occurred: %v", err)
+	desc := buildDescription(optionalDescription)
+	if desc != "" {
+		msg = fmt.Sprintf("%v (Description: %v)", msg, desc)
+	}
+	Logf(msg)
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	f.Flakes = append(f.Flakes, msg)
+	f.FlakeCount++
+}
+
+func (f *FlakeReport) GetFlakeCount() int {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return f.FlakeCount
+}
+
+func (f *FlakeReport) PrintHumanReadable() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	buf := bytes.Buffer{}
+	buf.WriteString(fmt.Sprintf("FlakeCount: %v\n", f.FlakeCount))
+	buf.WriteString("Flakes:\n")
+	for _, flake := range f.Flakes {
+		buf.WriteString(fmt.Sprintf("%v\n", flake))
+	}
+	return buf.String()
+}
+
+func (f *FlakeReport) PrintJSON() string {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+	return PrettyPrintJSON(f)
+}
+
+func (f *FlakeReport) SummaryKind() string {
+	return "FlakeReport"
+}

vendor/k8s.io/kubernetes/test/e2e/framework/framework.go

@@ -0,0 +1,879 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package framework contains provider-independent helper code for
+// building and running E2E tests with Ginkgo. The actual Ginkgo test
+// suites gets assembled by combining this framework, the optional
+// provider support code and specific tests via a separate .go file
+// like Kubernetes' test/e2e.go.
+package framework
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"math/rand"
+	"os"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/discovery"
+	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/dynamic"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/restmapper"
+	scaleclient "k8s.io/client-go/scale"
+	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
+	"k8s.io/kubernetes/test/e2e/framework/metrics"
+	testutils "k8s.io/kubernetes/test/utils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	maxKubectlExecRetries = 5
+	// TODO(mikedanese): reset this to 5 minutes once #47135 is resolved.
+	// ref https://github.com/kubernetes/kubernetes/issues/47135
+	DefaultNamespaceDeletionTimeout = 10 * time.Minute
+)
+
+// Framework supports common operations used by e2e tests; it will keep a client & a namespace for you.
+// Eventual goal is to merge this with integration test framework.
+type Framework struct {
+	BaseName string
+
+	// Set together with creating the ClientSet and the namespace.
+	// Guaranteed to be unique in the cluster even when running the same
+	// test multiple times in parallel.
+	UniqueName string
+
+	ClientSet                        clientset.Interface
+	KubemarkExternalClusterClientSet clientset.Interface
+
+	InternalClientset *internalclientset.Clientset
+	AggregatorClient  *aggregatorclient.Clientset
+	DynamicClient     dynamic.Interface
+
+	ScalesGetter scaleclient.ScalesGetter
+
+	SkipNamespaceCreation    bool            // Whether to skip creating a namespace
+	Namespace                *v1.Namespace   // Every test has at least one namespace unless creation is skipped
+	namespacesToDelete       []*v1.Namespace // Some tests have more than one.
+	NamespaceDeletionTimeout time.Duration
+	SkipPrivilegedPSPBinding bool // Whether to skip creating a binding to the privileged PSP in the test namespace
+
+	gatherer *ContainerResourceGatherer
+	// Constraints that passed to a check which is executed after data is gathered to
+	// see if 99% of results are within acceptable bounds. It has to be injected in the test,
+	// as expectations vary greatly. Constraints are grouped by the container names.
+	AddonResourceConstraints map[string]ResourceConstraint
+
+	logsSizeWaitGroup    sync.WaitGroup
+	logsSizeCloseChannel chan bool
+	logsSizeVerifier     *LogsSizeVerifier
+
+	// Flaky operation failures in an e2e test can be captured through this.
+	flakeReport *FlakeReport
+
+	// To make sure that this framework cleans up after itself, no matter what,
+	// we install a Cleanup action before each test and clear it after.  If we
+	// should abort, the AfterSuite hook should run all Cleanup actions.
+	cleanupHandle CleanupActionHandle
+
+	// configuration for framework's client
+	Options FrameworkOptions
+
+	// Place where various additional data is stored during test run to be printed to ReportDir,
+	// or stdout if ReportDir is not set once test ends.
+	TestSummaries []TestDataSummary
+
+	// Place to keep ClusterAutoscaler metrics from before test in order to compute delta.
+	clusterAutoscalerMetricsBeforeTest metrics.Collection
+}
+
+type TestDataSummary interface {
+	SummaryKind() string
+	PrintHumanReadable() string
+	PrintJSON() string
+}
+
+type FrameworkOptions struct {
+	ClientQPS    float32
+	ClientBurst  int
+	GroupVersion *schema.GroupVersion
+}
+
+// NewFramework makes a new framework and sets up a BeforeEach/AfterEach for
+// you (you can write additional before/after each functions).
+func NewDefaultFramework(baseName string) *Framework {
+	options := FrameworkOptions{
+		ClientQPS:   20,
+		ClientBurst: 50,
+	}
+	return NewFramework(baseName, options, nil)
+}
+
+func NewFramework(baseName string, options FrameworkOptions, client clientset.Interface) *Framework {
+	f := &Framework{
+		BaseName:                 baseName,
+		AddonResourceConstraints: make(map[string]ResourceConstraint),
+		Options:                  options,
+		ClientSet:                client,
+	}
+
+	BeforeEach(f.BeforeEach)
+	AfterEach(f.AfterEach)
+
+	return f
+}
+
+// BeforeEach gets a client and makes a namespace.
+func (f *Framework) BeforeEach() {
+	// The fact that we need this feels like a bug in ginkgo.
+	// https://github.com/onsi/ginkgo/issues/222
+	f.cleanupHandle = AddCleanupAction(f.AfterEach)
+	if f.ClientSet == nil {
+		By("Creating a kubernetes client")
+		config, err := LoadConfig()
+		testDesc := CurrentGinkgoTestDescription()
+		if len(testDesc.ComponentTexts) > 0 {
+			componentTexts := strings.Join(testDesc.ComponentTexts, " ")
+			config.UserAgent = fmt.Sprintf(
+				"%v -- %v",
+				rest.DefaultKubernetesUserAgent(),
+				componentTexts)
+		}
+
+		ExpectNoError(err)
+		config.QPS = f.Options.ClientQPS
+		config.Burst = f.Options.ClientBurst
+		if f.Options.GroupVersion != nil {
+			config.GroupVersion = f.Options.GroupVersion
+		}
+		if TestContext.KubeAPIContentType != "" {
+			config.ContentType = TestContext.KubeAPIContentType
+		}
+		f.ClientSet, err = clientset.NewForConfig(config)
+		ExpectNoError(err)
+		f.InternalClientset, err = internalclientset.NewForConfig(config)
+		ExpectNoError(err)
+		f.AggregatorClient, err = aggregatorclient.NewForConfig(config)
+		ExpectNoError(err)
+		f.DynamicClient, err = dynamic.NewForConfig(config)
+		ExpectNoError(err)
+		// node.k8s.io is based on CRD, which is served only as JSON
+		jsonConfig := config
+		jsonConfig.ContentType = "application/json"
+		ExpectNoError(err)
+
+		// create scales getter, set GroupVersion and NegotiatedSerializer to default values
+		// as they are required when creating a REST client.
+		if config.GroupVersion == nil {
+			config.GroupVersion = &schema.GroupVersion{}
+		}
+		if config.NegotiatedSerializer == nil {
+			config.NegotiatedSerializer = legacyscheme.Codecs
+		}
+		restClient, err := rest.RESTClientFor(config)
+		ExpectNoError(err)
+		discoClient, err := discovery.NewDiscoveryClientForConfig(config)
+		ExpectNoError(err)
+		cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoClient)
+		restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient)
+		restMapper.Reset()
+		resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient)
+		f.ScalesGetter = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver)
+
+		TestContext.CloudConfig.Provider.FrameworkBeforeEach(f)
+	}
+
+	if !f.SkipNamespaceCreation {
+		By(fmt.Sprintf("Building a namespace api object, basename %s", f.BaseName))
+		namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
+			"e2e-framework": f.BaseName,
+		})
+		ExpectNoError(err)
+
+		f.Namespace = namespace
+
+		if TestContext.VerifyServiceAccount {
+			By("Waiting for a default service account to be provisioned in namespace")
+			err = WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name)
+			ExpectNoError(err)
+		} else {
+			Logf("Skipping waiting for service account")
+		}
+		f.UniqueName = f.Namespace.GetName()
+	} else {
+		// not guaranteed to be unique, but very likely
+		f.UniqueName = fmt.Sprintf("%s-%08x", f.BaseName, rand.Int31())
+	}
+
+	if TestContext.GatherKubeSystemResourceUsageData != "false" && TestContext.GatherKubeSystemResourceUsageData != "none" {
+		var err error
+		var nodeMode NodesSet
+		switch TestContext.GatherKubeSystemResourceUsageData {
+		case "master":
+			nodeMode = MasterNodes
+		case "masteranddns":
+			nodeMode = MasterAndDNSNodes
+		default:
+			nodeMode = AllNodes
+		}
+
+		f.gatherer, err = NewResourceUsageGatherer(f.ClientSet, ResourceGathererOptions{
+			InKubemark:                  ProviderIs("kubemark"),
+			Nodes:                       nodeMode,
+			ResourceDataGatheringPeriod: 60 * time.Second,
+			ProbeDuration:               15 * time.Second,
+			PrintVerboseLogs:            false,
+		}, nil)
+		if err != nil {
+			Logf("Error while creating NewResourceUsageGatherer: %v", err)
+		} else {
+			go f.gatherer.StartGatheringData()
+		}
+	}
+
+	if TestContext.GatherLogsSizes {
+		f.logsSizeWaitGroup = sync.WaitGroup{}
+		f.logsSizeWaitGroup.Add(1)
+		f.logsSizeCloseChannel = make(chan bool)
+		f.logsSizeVerifier = NewLogsVerifier(f.ClientSet, f.logsSizeCloseChannel)
+		go func() {
+			f.logsSizeVerifier.Run()
+			f.logsSizeWaitGroup.Done()
+		}()
+	}
+
+	gatherMetricsAfterTest := TestContext.GatherMetricsAfterTest == "true" || TestContext.GatherMetricsAfterTest == "master"
+	if gatherMetricsAfterTest && TestContext.IncludeClusterAutoscalerMetrics {
+		grabber, err := metrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, !ProviderIs("kubemark"), false, false, false, TestContext.IncludeClusterAutoscalerMetrics)
+		if err != nil {
+			Logf("Failed to create MetricsGrabber (skipping ClusterAutoscaler metrics gathering before test): %v", err)
+		} else {
+			f.clusterAutoscalerMetricsBeforeTest, err = grabber.Grab()
+			if err != nil {
+				Logf("MetricsGrabber failed to grab CA metrics before test (skipping metrics gathering): %v", err)
+			} else {
+				Logf("Gathered ClusterAutoscaler metrics before test")
+			}
+		}
+
+	}
+
+	f.flakeReport = NewFlakeReport()
+}
+
+// AfterEach deletes the namespace, after reading its events.
+func (f *Framework) AfterEach() {
+	RemoveCleanupAction(f.cleanupHandle)
+
+	// DeleteNamespace at the very end in defer, to avoid any
+	// expectation failures preventing deleting the namespace.
+	defer func() {
+		nsDeletionErrors := map[string]error{}
+		// Whether to delete namespace is determined by 3 factors: delete-namespace flag, delete-namespace-on-failure flag and the test result
+		// if delete-namespace set to false, namespace will always be preserved.
+		// if delete-namespace is true and delete-namespace-on-failure is false, namespace will be preserved if test failed.
+		if TestContext.DeleteNamespace && (TestContext.DeleteNamespaceOnFailure || !CurrentGinkgoTestDescription().Failed) {
+			for _, ns := range f.namespacesToDelete {
+				By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name))
+				timeout := DefaultNamespaceDeletionTimeout
+				if f.NamespaceDeletionTimeout != 0 {
+					timeout = f.NamespaceDeletionTimeout
+				}
+				if err := deleteNS(f.ClientSet, f.DynamicClient, ns.Name, timeout); err != nil {
+					if !apierrors.IsNotFound(err) {
+						nsDeletionErrors[ns.Name] = err
+					} else {
+						Logf("Namespace %v was already deleted", ns.Name)
+					}
+				}
+			}
+		} else {
+			if !TestContext.DeleteNamespace {
+				Logf("Found DeleteNamespace=false, skipping namespace deletion!")
+			} else {
+				Logf("Found DeleteNamespaceOnFailure=false and current test failed, skipping namespace deletion!")
+			}
+		}
+
+		// Paranoia-- prevent reuse!
+		f.Namespace = nil
+		f.ClientSet = nil
+		f.namespacesToDelete = nil
+
+		// if we had errors deleting, report them now.
+		if len(nsDeletionErrors) != 0 {
+			messages := []string{}
+			for namespaceKey, namespaceErr := range nsDeletionErrors {
+				messages = append(messages, fmt.Sprintf("Couldn't delete ns: %q: %s (%#v)", namespaceKey, namespaceErr, namespaceErr))
+			}
+			Failf(strings.Join(messages, ","))
+		}
+	}()
+
+	// Print events if the test failed.
+	if CurrentGinkgoTestDescription().Failed && TestContext.DumpLogsOnFailure {
+		// Pass both unversioned client and versioned clientset, till we have removed all uses of the unversioned client.
+		if !f.SkipNamespaceCreation {
+			DumpAllNamespaceInfo(f.ClientSet, f.Namespace.Name)
+		}
+	}
+
+	if TestContext.GatherKubeSystemResourceUsageData != "false" && TestContext.GatherKubeSystemResourceUsageData != "none" && f.gatherer != nil {
+		By("Collecting resource usage data")
+		summary, resourceViolationError := f.gatherer.StopAndSummarize([]int{90, 99, 100}, f.AddonResourceConstraints)
+		defer ExpectNoError(resourceViolationError)
+		f.TestSummaries = append(f.TestSummaries, summary)
+	}
+
+	if TestContext.GatherLogsSizes {
+		By("Gathering log sizes data")
+		close(f.logsSizeCloseChannel)
+		f.logsSizeWaitGroup.Wait()
+		f.TestSummaries = append(f.TestSummaries, f.logsSizeVerifier.GetSummary())
+	}
+
+	if TestContext.GatherMetricsAfterTest != "false" {
+		By("Gathering metrics")
+		// Grab apiserver, scheduler, controller-manager metrics and (optionally) nodes' kubelet metrics.
+		grabMetricsFromKubelets := TestContext.GatherMetricsAfterTest != "master" && !ProviderIs("kubemark")
+		grabber, err := metrics.NewMetricsGrabber(f.ClientSet, f.KubemarkExternalClusterClientSet, grabMetricsFromKubelets, true, true, true, TestContext.IncludeClusterAutoscalerMetrics)
+		if err != nil {
+			Logf("Failed to create MetricsGrabber (skipping metrics gathering): %v", err)
+		} else {
+			received, err := grabber.Grab()
+			if err != nil {
+				Logf("MetricsGrabber failed to grab some of the metrics: %v", err)
+			}
+			(*MetricsForE2E)(&received).computeClusterAutoscalerMetricsDelta(f.clusterAutoscalerMetricsBeforeTest)
+			f.TestSummaries = append(f.TestSummaries, (*MetricsForE2E)(&received))
+		}
+	}
+
+	TestContext.CloudConfig.Provider.FrameworkAfterEach(f)
+
+	// Report any flakes that were observed in the e2e test and reset.
+	if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 {
+		f.TestSummaries = append(f.TestSummaries, f.flakeReport)
+		f.flakeReport = nil
+	}
+
+	PrintSummaries(f.TestSummaries, f.BaseName)
+
+	// Check whether all nodes are ready after the test.
+	// This is explicitly done at the very end of the test, to avoid
+	// e.g. not removing namespace in case of this failure.
+	if err := AllNodesReady(f.ClientSet, 3*time.Minute); err != nil {
+		Failf("All nodes should be ready after test, %v", err)
+	}
+}
+
+func (f *Framework) CreateNamespace(baseName string, labels map[string]string) (*v1.Namespace, error) {
+	createTestingNS := TestContext.CreateTestingNS
+	if createTestingNS == nil {
+		createTestingNS = CreateTestingNS
+	}
+	ns, err := createTestingNS(baseName, f.ClientSet, labels)
+	// check ns instead of err to see if it's nil as we may
+	// fail to create serviceAccount in it.
+	f.AddNamespacesToDelete(ns)
+
+	if err == nil && !f.SkipPrivilegedPSPBinding {
+		CreatePrivilegedPSPBinding(f, ns.Name)
+	}
+
+	return ns, err
+}
+
+func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) {
+	f.flakeReport.RecordFlakeIfError(err, optionalDescription)
+}
+
+// AddNamespacesToDelete adds one or more namespaces to be deleted when the test
+// completes.
+func (f *Framework) AddNamespacesToDelete(namespaces ...*v1.Namespace) {
+	for _, ns := range namespaces {
+		if ns == nil {
+			continue
+		}
+		f.namespacesToDelete = append(f.namespacesToDelete, ns)
+
+	}
+}
+
+// WaitForPodTerminated waits for the pod to be terminated with the given reason.
+func (f *Framework) WaitForPodTerminated(podName, reason string) error {
+	return waitForPodTerminatedInNamespace(f.ClientSet, podName, reason, f.Namespace.Name)
+}
+
+// WaitForPodNotFound waits for the pod to be completely terminated (not "Get-able").
+func (f *Framework) WaitForPodNotFound(podName string, timeout time.Duration) error {
+	return waitForPodNotFoundInNamespace(f.ClientSet, podName, f.Namespace.Name, timeout)
+}
+
+// WaitForPodRunning waits for the pod to run in the namespace.
+func (f *Framework) WaitForPodRunning(podName string) error {
+	return WaitForPodNameRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
+}
+
+// WaitForPodReady waits for the pod to flip to ready in the namespace.
+func (f *Framework) WaitForPodReady(podName string) error {
+	return waitTimeoutForPodReadyInNamespace(f.ClientSet, podName, f.Namespace.Name, PodStartTimeout)
+}
+
+// WaitForPodRunningSlow waits for the pod to run in the namespace.
+// It has a longer timeout then WaitForPodRunning (util.slowPodStartTimeout).
+func (f *Framework) WaitForPodRunningSlow(podName string) error {
+	return waitForPodRunningInNamespaceSlow(f.ClientSet, podName, f.Namespace.Name)
+}
+
+// WaitForPodNoLongerRunning waits for the pod to no longer be running in the namespace, for either
+// success or failure.
+func (f *Framework) WaitForPodNoLongerRunning(podName string) error {
+	return WaitForPodNoLongerRunningInNamespace(f.ClientSet, podName, f.Namespace.Name)
+}
+
+// TestContainerOutput runs the given pod in the given namespace and waits
+// for all of the containers in the podSpec to move into the 'Success' status, and tests
+// the specified container log against the given expected output using a substring matcher.
+func (f *Framework) TestContainerOutput(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
+	f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, ContainSubstring)
+}
+
+// TestContainerOutputRegexp runs the given pod in the given namespace and waits
+// for all of the containers in the podSpec to move into the 'Success' status, and tests
+// the specified container log against the given expected output using a regexp matcher.
+func (f *Framework) TestContainerOutputRegexp(scenarioName string, pod *v1.Pod, containerIndex int, expectedOutput []string) {
+	f.testContainerOutputMatcher(scenarioName, pod, containerIndex, expectedOutput, MatchRegexp)
+}
+
+// Write a file using kubectl exec echo <contents> > <path> via specified container
+// Because of the primitive technique we're using here, we only allow ASCII alphanumeric characters
+func (f *Framework) WriteFileViaContainer(podName, containerName string, path string, contents string) error {
+	By("writing a file in the container")
+	allowedCharacters := "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+	for _, c := range contents {
+		if !strings.ContainsRune(allowedCharacters, c) {
+			return fmt.Errorf("Unsupported character in string to write: %v", c)
+		}
+	}
+	command := fmt.Sprintf("echo '%s' > '%s'", contents, path)
+	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "/bin/sh", "-c", command)
+	if err != nil {
+		Logf("error running kubectl exec to write file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
+	}
+	return err
+}
+
+// Read a file using kubectl exec cat <path>
+func (f *Framework) ReadFileViaContainer(podName, containerName string, path string) (string, error) {
+	By("reading a file in the container")
+
+	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "cat", path)
+	if err != nil {
+		Logf("error running kubectl exec to read file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
+	}
+	return string(stdout), err
+}
+
+func (f *Framework) CheckFileSizeViaContainer(podName, containerName, path string) (string, error) {
+	By("checking a file size in the container")
+
+	stdout, stderr, err := kubectlExecWithRetry(f.Namespace.Name, podName, containerName, "--", "ls", "-l", path)
+	if err != nil {
+		Logf("error running kubectl exec to read file: %v\nstdout=%v\nstderr=%v)", err, string(stdout), string(stderr))
+	}
+	return string(stdout), err
+}
+
+// CreateServiceForSimpleAppWithPods is a convenience wrapper to create a service and its matching pods all at once.
+func (f *Framework) CreateServiceForSimpleAppWithPods(contPort int, svcPort int, appName string, podSpec func(n v1.Node) v1.PodSpec, count int, block bool) (error, *v1.Service) {
+	var err error = nil
+	theService := f.CreateServiceForSimpleApp(contPort, svcPort, appName)
+	f.CreatePodsPerNodeForSimpleApp(appName, podSpec, count)
+	if block {
+		err = testutils.WaitForPodsWithLabelRunning(f.ClientSet, f.Namespace.Name, labels.SelectorFromSet(labels.Set(theService.Spec.Selector)))
+	}
+	return err, theService
+}
+
+// CreateServiceForSimpleApp returns a service that selects/exposes pods (send -1 ports if no exposure needed) with an app label.
+func (f *Framework) CreateServiceForSimpleApp(contPort, svcPort int, appName string) *v1.Service {
+	if appName == "" {
+		panic(fmt.Sprintf("no app name provided"))
+	}
+
+	serviceSelector := map[string]string{
+		"app": appName + "-pod",
+	}
+
+	// For convenience, user sending ports are optional.
+	portsFunc := func() []v1.ServicePort {
+		if contPort < 1 || svcPort < 1 {
+			return nil
+		} else {
+			return []v1.ServicePort{{
+				Protocol:   v1.ProtocolTCP,
+				Port:       int32(svcPort),
+				TargetPort: intstr.FromInt(contPort),
+			}}
+		}
+	}
+	Logf("Creating a service-for-%v for selecting app=%v-pod", appName, appName)
+	service, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(&v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "service-for-" + appName,
+			Labels: map[string]string{
+				"app": appName + "-service",
+			},
+		},
+		Spec: v1.ServiceSpec{
+			Ports:    portsFunc(),
+			Selector: serviceSelector,
+		},
+	})
+	ExpectNoError(err)
+	return service
+}
+
+// CreatePodsPerNodeForSimpleApp Creates pods w/ labels.  Useful for tests which make a bunch of pods w/o any networking.
+func (f *Framework) CreatePodsPerNodeForSimpleApp(appName string, podSpec func(n v1.Node) v1.PodSpec, maxCount int) map[string]string {
+	nodes := GetReadySchedulableNodesOrDie(f.ClientSet)
+	labels := map[string]string{
+		"app": appName + "-pod",
+	}
+	for i, node := range nodes.Items {
+		// one per node, but no more than maxCount.
+		if i <= maxCount {
+			Logf("%v/%v : Creating container with label app=%v-pod", i, maxCount, appName)
+			_, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(&v1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:   fmt.Sprintf(appName+"-pod-%v", i),
+					Labels: labels,
+				},
+				Spec: podSpec(node),
+			})
+			ExpectNoError(err)
+		}
+	}
+	return labels
+}
+
+type KubeUser struct {
+	Name string `yaml:"name"`
+	User struct {
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
+		Token    string `yaml:"token"`
+	} `yaml:"user"`
+}
+
+type KubeCluster struct {
+	Name    string `yaml:"name"`
+	Cluster struct {
+		CertificateAuthorityData string `yaml:"certificate-authority-data"`
+		Server                   string `yaml:"server"`
+	} `yaml:"cluster"`
+}
+
+type KubeConfig struct {
+	Contexts []struct {
+		Name    string `yaml:"name"`
+		Context struct {
+			Cluster string `yaml:"cluster"`
+			User    string
+		} `yaml:"context"`
+	} `yaml:"contexts"`
+
+	Clusters []KubeCluster `yaml:"clusters"`
+
+	Users []KubeUser `yaml:"users"`
+}
+
+func (kc *KubeConfig) FindUser(name string) *KubeUser {
+	for _, user := range kc.Users {
+		if user.Name == name {
+			return &user
+		}
+	}
+	return nil
+}
+
+func (kc *KubeConfig) FindCluster(name string) *KubeCluster {
+	for _, cluster := range kc.Clusters {
+		if cluster.Name == name {
+			return &cluster
+		}
+	}
+	return nil
+}
+
+func kubectlExecWithRetry(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
+	for numRetries := 0; numRetries < maxKubectlExecRetries; numRetries++ {
+		if numRetries > 0 {
+			Logf("Retrying kubectl exec (retry count=%v/%v)", numRetries+1, maxKubectlExecRetries)
+		}
+
+		stdOutBytes, stdErrBytes, err := kubectlExec(namespace, podName, containerName, args...)
+		if err != nil {
+			if strings.Contains(strings.ToLower(string(stdErrBytes)), "i/o timeout") {
+				// Retry on "i/o timeout" errors
+				Logf("Warning: kubectl exec encountered i/o timeout.\nerr=%v\nstdout=%v\nstderr=%v)", err, string(stdOutBytes), string(stdErrBytes))
+				continue
+			}
+			if strings.Contains(strings.ToLower(string(stdErrBytes)), "container not found") {
+				// Retry on "container not found" errors
+				Logf("Warning: kubectl exec encountered container not found.\nerr=%v\nstdout=%v\nstderr=%v)", err, string(stdOutBytes), string(stdErrBytes))
+				time.Sleep(2 * time.Second)
+				continue
+			}
+		}
+
+		return stdOutBytes, stdErrBytes, err
+	}
+	err := fmt.Errorf("Failed: kubectl exec failed %d times with \"i/o timeout\". Giving up.", maxKubectlExecRetries)
+	return nil, nil, err
+}
+
+func kubectlExec(namespace string, podName, containerName string, args ...string) ([]byte, []byte, error) {
+	var stdout, stderr bytes.Buffer
+	cmdArgs := []string{
+		"exec",
+		fmt.Sprintf("--namespace=%v", namespace),
+		podName,
+		fmt.Sprintf("-c=%v", containerName),
+	}
+	cmdArgs = append(cmdArgs, args...)
+
+	cmd := KubectlCmd(cmdArgs...)
+	cmd.Stdout, cmd.Stderr = &stdout, &stderr
+
+	Logf("Running '%s %s'", cmd.Path, strings.Join(cmdArgs, " "))
+	err := cmd.Run()
+	return stdout.Bytes(), stderr.Bytes(), err
+}
+
+// Wrapper function for ginkgo describe.  Adds namespacing.
+// TODO: Support type safe tagging as well https://github.com/kubernetes/kubernetes/pull/22401.
+func KubeDescribe(text string, body func()) bool {
+	return Describe("[k8s.io] "+text, body)
+}
+
+// Wrapper function for ginkgo It.  Adds "[Conformance]" tag and makes static analysis easier.
+func ConformanceIt(text string, body interface{}, timeout ...float64) bool {
+	return It(text+" [Conformance]", body, timeout...)
+}
+
+// PodStateVerification represents a verification of pod state.
+// Any time you have a set of pods that you want to operate against or query,
+// this struct can be used to declaratively identify those pods.
+type PodStateVerification struct {
+	// Optional: only pods that have k=v labels will pass this filter.
+	Selectors map[string]string
+
+	// Required: The phases which are valid for your pod.
+	ValidPhases []v1.PodPhase
+
+	// Optional: only pods passing this function will pass the filter
+	// Verify a pod.
+	// As an optimization, in addition to specifying filter (boolean),
+	// this function allows specifying an error as well.
+	// The error indicates that the polling of the pod spectrum should stop.
+	Verify func(v1.Pod) (bool, error)
+
+	// Optional: only pods with this name will pass the filter.
+	PodName string
+}
+
+type ClusterVerification struct {
+	client    clientset.Interface
+	namespace *v1.Namespace // pointer rather than string, since ns isn't created until before each.
+	podState  PodStateVerification
+}
+
+func (f *Framework) NewClusterVerification(namespace *v1.Namespace, filter PodStateVerification) *ClusterVerification {
+	return &ClusterVerification{
+		f.ClientSet,
+		namespace,
+		filter,
+	}
+}
+
+func passesPodNameFilter(pod v1.Pod, name string) bool {
+	return name == "" || strings.Contains(pod.Name, name)
+}
+
+func passesVerifyFilter(pod v1.Pod, verify func(p v1.Pod) (bool, error)) (bool, error) {
+	if verify == nil {
+		return true, nil
+	} else {
+		verified, err := verify(pod)
+		// If an error is returned, by definition, pod verification fails
+		if err != nil {
+			return false, err
+		} else {
+			return verified, nil
+		}
+	}
+}
+
+func passesPhasesFilter(pod v1.Pod, validPhases []v1.PodPhase) bool {
+	passesPhaseFilter := false
+	for _, phase := range validPhases {
+		if pod.Status.Phase == phase {
+			passesPhaseFilter = true
+		}
+	}
+	return passesPhaseFilter
+}
+
+// filterLabels returns a list of pods which have labels.
+func filterLabels(selectors map[string]string, cli clientset.Interface, ns string) (*v1.PodList, error) {
+	var err error
+	var selector labels.Selector
+	var pl *v1.PodList
+	// List pods based on selectors.  This might be a tiny optimization rather then filtering
+	// everything manually.
+	if len(selectors) > 0 {
+		selector = labels.SelectorFromSet(labels.Set(selectors))
+		options := metav1.ListOptions{LabelSelector: selector.String()}
+		pl, err = cli.CoreV1().Pods(ns).List(options)
+	} else {
+		pl, err = cli.CoreV1().Pods(ns).List(metav1.ListOptions{})
+	}
+	return pl, err
+}
+
+// filter filters pods which pass a filter.  It can be used to compose
+// the more useful abstractions like ForEach, WaitFor, and so on, which
+// can be used directly by tests.
+func (p *PodStateVerification) filter(c clientset.Interface, namespace *v1.Namespace) ([]v1.Pod, error) {
+	if len(p.ValidPhases) == 0 || namespace == nil {
+		panic(fmt.Errorf("Need to specify a valid pod phases (%v) and namespace (%v). ", p.ValidPhases, namespace))
+	}
+
+	ns := namespace.Name
+	pl, err := filterLabels(p.Selectors, c, ns) // Build an v1.PodList to operate against.
+	Logf("Selector matched %v pods for %v", len(pl.Items), p.Selectors)
+	if len(pl.Items) == 0 || err != nil {
+		return pl.Items, err
+	}
+
+	unfilteredPods := pl.Items
+	filteredPods := []v1.Pod{}
+ReturnPodsSoFar:
+	// Next: Pod must match at least one of the states that the user specified
+	for _, pod := range unfilteredPods {
+		if !(passesPhasesFilter(pod, p.ValidPhases) && passesPodNameFilter(pod, p.PodName)) {
+			continue
+		}
+		passesVerify, err := passesVerifyFilter(pod, p.Verify)
+		if err != nil {
+			Logf("Error detected on %v : %v !", pod.Name, err)
+			break ReturnPodsSoFar
+		}
+		if passesVerify {
+			filteredPods = append(filteredPods, pod)
+		}
+	}
+	return filteredPods, err
+}
+
+// WaitFor waits for some minimum number of pods to be verified, according to the PodStateVerification
+// definition.
+func (cl *ClusterVerification) WaitFor(atLeast int, timeout time.Duration) ([]v1.Pod, error) {
+	pods := []v1.Pod{}
+	var returnedErr error
+
+	err := wait.Poll(1*time.Second, timeout, func() (bool, error) {
+		pods, returnedErr = cl.podState.filter(cl.client, cl.namespace)
+
+		// Failure
+		if returnedErr != nil {
+			Logf("Cutting polling short: We got an error from the pod filtering layer.")
+			// stop polling if the pod filtering returns an error.  that should never happen.
+			// it indicates, for example, that the client is broken or something non-pod related.
+			return false, returnedErr
+		}
+		Logf("Found %v / %v", len(pods), atLeast)
+
+		// Success
+		if len(pods) >= atLeast {
+			return true, nil
+		}
+		// Keep trying...
+		return false, nil
+	})
+	Logf("WaitFor completed with timeout %v.  Pods found = %v out of %v", timeout, len(pods), atLeast)
+	return pods, err
+}
+
+// WaitForOrFail provides a shorthand WaitFor with failure as an option if anything goes wrong.
+func (cl *ClusterVerification) WaitForOrFail(atLeast int, timeout time.Duration) {
+	pods, err := cl.WaitFor(atLeast, timeout)
+	if err != nil || len(pods) < atLeast {
+		Failf("Verified %v of %v pods , error : %v", len(pods), atLeast, err)
+	}
+}
+
+// ForEach runs a function against every verifiable pod.  Be warned that this doesn't wait for "n" pods to verify,
+// so it may return very quickly if you have strict pod state requirements.
+//
+// For example, if you require at least 5 pods to be running before your test will pass,
+// its smart to first call "clusterVerification.WaitFor(5)" before you call clusterVerification.ForEach.
+func (cl *ClusterVerification) ForEach(podFunc func(v1.Pod)) error {
+	pods, err := cl.podState.filter(cl.client, cl.namespace)
+	if err == nil {
+		if len(pods) == 0 {
+			Failf("No pods matched the filter.")
+		}
+		Logf("ForEach: Found %v pods from the filter.  Now looping through them.", len(pods))
+		for _, p := range pods {
+			podFunc(p)
+		}
+	} else {
+		Logf("ForEach: Something went wrong when filtering pods to execute against: %v", err)
+	}
+
+	return err
+}
+
+// GetLogToFileFunc is a convenience function that returns a function that have the same interface as
+// Logf, but writes to a specified file.
+func GetLogToFileFunc(file *os.File) func(format string, args ...interface{}) {
+	return func(format string, args ...interface{}) {
+		writer := bufio.NewWriter(file)
+		if _, err := fmt.Fprintf(writer, format, args...); err != nil {
+			Logf("Failed to write file %v with test performance data: %v", file.Name(), err)
+		}
+		writer.Flush()
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/get-kubemark-resource-usage.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bufio"
+	"fmt"
+	"strings"
+)
+
+type KubemarkResourceUsage struct {
+	Name                    string
+	MemoryWorkingSetInBytes uint64
+	CPUUsageInCores         float64
+}
+
+func getMasterUsageByPrefix(prefix string) (string, error) {
+	sshResult, err := SSH(fmt.Sprintf("ps ax -o %%cpu,rss,command | tail -n +2 | grep %v | sed 's/\\s+/ /g'", prefix), GetMasterHost()+":22", TestContext.Provider)
+	if err != nil {
+		return "", err
+	}
+	return sshResult.Stdout, nil
+}
+
+// TODO: figure out how to move this to kubemark directory (need to factor test SSH out of e2e framework)
+func GetKubemarkMasterComponentsResourceUsage() map[string]*KubemarkResourceUsage {
+	result := make(map[string]*KubemarkResourceUsage)
+	// Get kubernetes component resource usage
+	sshResult, err := getMasterUsageByPrefix("kube")
+	if err != nil {
+		Logf("Error when trying to SSH to master machine. Skipping probe. %v", err)
+		return nil
+	}
+	scanner := bufio.NewScanner(strings.NewReader(sshResult))
+	for scanner.Scan() {
+		var cpu float64
+		var mem uint64
+		var name string
+		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /usr/local/bin/kube-%s", &cpu, &mem, &name)
+		if name != "" {
+			// Gatherer expects pod_name/container_name format
+			fullName := name + "/" + name
+			result[fullName] = &KubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
+		}
+	}
+	// Get etcd resource usage
+	sshResult, err = getMasterUsageByPrefix("bin/etcd")
+	if err != nil {
+		Logf("Error when trying to SSH to master machine. Skipping probe")
+		return nil
+	}
+	scanner = bufio.NewScanner(strings.NewReader(sshResult))
+	for scanner.Scan() {
+		var cpu float64
+		var mem uint64
+		var etcdKind string
+		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d /bin/sh -c /usr/local/bin/etcd", &cpu, &mem)
+		dataDirStart := strings.Index(scanner.Text(), "--data-dir")
+		if dataDirStart < 0 {
+			continue
+		}
+		fmt.Sscanf(scanner.Text()[dataDirStart:], "--data-dir=/var/%s", &etcdKind)
+		if etcdKind != "" {
+			// Gatherer expects pod_name/container_name format
+			fullName := "etcd/" + etcdKind
+			result[fullName] = &KubemarkResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
+		}
+	}
+	return result
+}

vendor/k8s.io/kubernetes/test/e2e/framework/ginkgowrapper/wrapper.go

@@ -0,0 +1,134 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package ginkgowrapper wraps Ginkgo Fail and Skip functions to panic
+// with structured data instead of a constant string.
+package ginkgowrapper
+
+import (
+	"bufio"
+	"bytes"
+	"regexp"
+	"runtime"
+	"runtime/debug"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+)
+
+// FailurePanic is the value that will be panicked from Fail.
+type FailurePanic struct {
+	Message        string // The failure message passed to Fail
+	Filename       string // The filename that is the source of the failure
+	Line           int    // The line number of the filename that is the source of the failure
+	FullStackTrace string // A full stack trace starting at the source of the failure
+}
+
+// String makes FailurePanic look like the old Ginkgo panic when printed.
+func (FailurePanic) String() string { return ginkgo.GINKGO_PANIC }
+
+// Fail wraps ginkgo.Fail so that it panics with more useful
+// information about the failure. This function will panic with a
+// FailurePanic.
+func Fail(message string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+
+	_, file, line, _ := runtime.Caller(skip)
+	fp := FailurePanic{
+		Message:        message,
+		Filename:       file,
+		Line:           line,
+		FullStackTrace: pruneStack(skip),
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			panic(fp)
+		}
+	}()
+
+	ginkgo.Fail(message, skip)
+}
+
+// SkipPanic is the value that will be panicked from Skip.
+type SkipPanic struct {
+	Message        string // The failure message passed to Fail
+	Filename       string // The filename that is the source of the failure
+	Line           int    // The line number of the filename that is the source of the failure
+	FullStackTrace string // A full stack trace starting at the source of the failure
+}
+
+// String makes SkipPanic look like the old Ginkgo panic when printed.
+func (SkipPanic) String() string { return ginkgo.GINKGO_PANIC }
+
+// Skip wraps ginkgo.Skip so that it panics with more useful
+// information about why the test is being skipped. This function will
+// panic with a SkipPanic.
+func Skip(message string, callerSkip ...int) {
+	skip := 1
+	if len(callerSkip) > 0 {
+		skip += callerSkip[0]
+	}
+
+	_, file, line, _ := runtime.Caller(skip)
+	sp := SkipPanic{
+		Message:        message,
+		Filename:       file,
+		Line:           line,
+		FullStackTrace: pruneStack(skip),
+	}
+
+	defer func() {
+		e := recover()
+		if e != nil {
+			panic(sp)
+		}
+	}()
+
+	ginkgo.Skip(message, skip)
+}
+
+// ginkgo adds a lot of test running infrastructure to the stack, so
+// we filter those out
+var stackSkipPattern = regexp.MustCompile(`onsi/ginkgo`)
+
+func pruneStack(skip int) string {
+	skip += 2 // one for pruneStack and one for debug.Stack
+	stack := debug.Stack()
+	scanner := bufio.NewScanner(bytes.NewBuffer(stack))
+	var prunedStack []string
+
+	// skip the top of the stack
+	for i := 0; i < 2*skip+1; i++ {
+		scanner.Scan()
+	}
+
+	for scanner.Scan() {
+		if stackSkipPattern.Match(scanner.Bytes()) {
+			scanner.Scan() // these come in pairs
+		} else {
+			prunedStack = append(prunedStack, scanner.Text())
+			scanner.Scan() // these come in pairs
+			prunedStack = append(prunedStack, scanner.Text())
+		}
+	}
+
+	return strings.Join(prunedStack, "\n")
+}

vendor/k8s.io/kubernetes/test/e2e/framework/google_compute.go

@@ -0,0 +1,208 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+// TODO: These should really just use the GCE API client library or at least use
+// better formatted output from the --format flag.
+
+// Returns master & node image string, or error
+func lookupClusterImageSources() (string, string, error) {
+	// Given args for a gcloud compute command, run it with other args, and return the values,
+	// whether separated by newlines, commas or semicolons.
+	gcloudf := func(argv ...string) ([]string, error) {
+		args := []string{"compute"}
+		args = append(args, argv...)
+		args = append(args, "--project", TestContext.CloudConfig.ProjectID)
+		if TestContext.CloudConfig.MultiMaster {
+			args = append(args, "--region", TestContext.CloudConfig.Region)
+		} else {
+			args = append(args, "--zone", TestContext.CloudConfig.Zone)
+		}
+		outputBytes, err := exec.Command("gcloud", args...).CombinedOutput()
+		str := strings.Replace(string(outputBytes), ",", "\n", -1)
+		str = strings.Replace(str, ";", "\n", -1)
+		lines := strings.Split(str, "\n")
+		if err != nil {
+			Logf("lookupDiskImageSources: gcloud error with [%#v]; err:%v", argv, err)
+			for _, l := range lines {
+				Logf(" > %s", l)
+			}
+		}
+		return lines, err
+	}
+
+	// Given a GCE instance, look through its disks, finding one that has a sourceImage
+	host2image := func(instance string) (string, error) {
+		// gcloud compute instances describe {INSTANCE} --format="get(disks[].source)"
+		// gcloud compute disks describe {DISKURL} --format="get(sourceImage)"
+		disks, err := gcloudf("instances", "describe", instance, "--format=get(disks[].source)")
+		if err != nil {
+			return "", err
+		} else if len(disks) == 0 {
+			return "", fmt.Errorf("instance %q had no findable disks", instance)
+		}
+		// Loop over disks, looking for the boot disk
+		for _, disk := range disks {
+			lines, err := gcloudf("disks", "describe", disk, "--format=get(sourceImage)")
+			if err != nil {
+				return "", err
+			} else if len(lines) > 0 && lines[0] != "" {
+				return lines[0], nil // break, we're done
+			}
+		}
+		return "", fmt.Errorf("instance %q had no disk with a sourceImage", instance)
+	}
+
+	// gcloud compute instance-groups list-instances {GROUPNAME} --format="get(instance)"
+	nodeName := ""
+	instGroupName := strings.Split(TestContext.CloudConfig.NodeInstanceGroup, ",")[0]
+	if lines, err := gcloudf("instance-groups", "list-instances", instGroupName, "--format=get(instance)"); err != nil {
+		return "", "", err
+	} else if len(lines) == 0 {
+		return "", "", fmt.Errorf("no instances inside instance-group %q", instGroupName)
+	} else {
+		nodeName = lines[0]
+	}
+
+	nodeImg, err := host2image(nodeName)
+	if err != nil {
+		return "", "", err
+	}
+	frags := strings.Split(nodeImg, "/")
+	nodeImg = frags[len(frags)-1]
+
+	// For GKE clusters, MasterName will not be defined; we just leave masterImg blank.
+	masterImg := ""
+	if masterName := TestContext.CloudConfig.MasterName; masterName != "" {
+		img, err := host2image(masterName)
+		if err != nil {
+			return "", "", err
+		}
+		frags = strings.Split(img, "/")
+		masterImg = frags[len(frags)-1]
+	}
+
+	return masterImg, nodeImg, nil
+}
+
+func LogClusterImageSources() {
+	masterImg, nodeImg, err := lookupClusterImageSources()
+	if err != nil {
+		Logf("Cluster image sources lookup failed: %v\n", err)
+		return
+	}
+	Logf("cluster-master-image: %s", masterImg)
+	Logf("cluster-node-image: %s", nodeImg)
+
+	images := map[string]string{
+		"master_os_image": masterImg,
+		"node_os_image":   nodeImg,
+	}
+
+	outputBytes, _ := json.MarshalIndent(images, "", "  ")
+	filePath := filepath.Join(TestContext.ReportDir, "images.json")
+	if err := ioutil.WriteFile(filePath, outputBytes, 0644); err != nil {
+		Logf("cluster images sources, could not write to %q: %v", filePath, err)
+	}
+}
+
+func CreateManagedInstanceGroup(size int64, zone, template string) error {
+	// TODO(verult): make this hit the compute API directly instead of
+	// shelling out to gcloud.
+	_, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
+		"create",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		fmt.Sprintf("--zone=%s", zone),
+		TestContext.CloudConfig.NodeInstanceGroup,
+		fmt.Sprintf("--size=%d", size),
+		fmt.Sprintf("--template=%s", template))
+	if err != nil {
+		return fmt.Errorf("gcloud compute instance-groups managed create call failed with err: %v", err)
+	}
+	return nil
+}
+
+func GetManagedInstanceGroupTemplateName(zone string) (string, error) {
+	// TODO(verult): make this hit the compute API directly instead of
+	// shelling out to gcloud. Use InstanceGroupManager to get Instance Template name.
+
+	stdout, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
+		"list",
+		fmt.Sprintf("--filter=name:%s", TestContext.CloudConfig.NodeInstanceGroup),
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		fmt.Sprintf("--zones=%s", zone),
+	)
+
+	if err != nil {
+		return "", fmt.Errorf("gcloud compute instance-groups managed list call failed with err: %v", err)
+	}
+
+	templateName, err := parseInstanceTemplateName(stdout)
+	if err != nil {
+		return "", fmt.Errorf("error parsing gcloud output: %v", err)
+	}
+	return templateName, nil
+}
+
+func DeleteManagedInstanceGroup(zone string) error {
+	// TODO(verult): make this hit the compute API directly instead of
+	// shelling out to gcloud.
+	_, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
+		"delete",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		fmt.Sprintf("--zone=%s", zone),
+		TestContext.CloudConfig.NodeInstanceGroup)
+	if err != nil {
+		return fmt.Errorf("gcloud compute instance-groups managed delete call failed with err: %v", err)
+	}
+	return nil
+}
+
+func parseInstanceTemplateName(gcloudOutput string) (string, error) {
+	const templateNameField = "INSTANCE_TEMPLATE"
+
+	lines := strings.Split(gcloudOutput, "\n")
+	if len(lines) <= 1 { // Empty output or only contains column names
+		return "", fmt.Errorf("the list is empty")
+	}
+
+	// Otherwise, there should be exactly 1 entry, i.e. 2 lines
+	fieldNames := strings.Fields(lines[0])
+	instanceTemplateColumn := 0
+	for instanceTemplateColumn < len(fieldNames) &&
+		fieldNames[instanceTemplateColumn] != templateNameField {
+		instanceTemplateColumn++
+	}
+
+	if instanceTemplateColumn == len(fieldNames) {
+		return "", fmt.Errorf("the list does not contain instance template information")
+	}
+
+	fields := strings.Fields(lines[1])
+	instanceTemplateName := fields[instanceTemplateColumn]
+
+	return instanceTemplateName, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/gpu_util.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/klog"
+)
+
+const (
+	// GPUResourceName is the extended name of the GPU resource since v1.8
+	// this uses the device plugin mechanism
+	NVIDIAGPUResourceName = "nvidia.com/gpu"
+
+	// TODO: Parametrize it by making it a feature in TestFramework.
+	// so we can override the daemonset in other setups (non COS).
+	// GPUDevicePluginDSYAML is the official Google Device Plugin Daemonset NVIDIA GPU manifest for GKE
+	GPUDevicePluginDSYAML = "https://raw.githubusercontent.com/kubernetes/kubernetes/master/cluster/addons/device-plugins/nvidia-gpu/daemonset.yaml"
+)
+
+// TODO make this generic and not linked to COS only
+// NumberOfGPUs returs the number of GPUs advertised by a node
+// This is based on the Device Plugin system and expected to run on a COS based node
+// After the NVIDIA drivers were installed
+func NumberOfNVIDIAGPUs(node *v1.Node) int64 {
+	val, ok := node.Status.Capacity[NVIDIAGPUResourceName]
+
+	if !ok {
+		return 0
+	}
+
+	return val.Value()
+}
+
+// NVIDIADevicePlugin returns the official Google Device Plugin pod for NVIDIA GPU in GKE
+func NVIDIADevicePlugin() *v1.Pod {
+	ds, err := DsFromManifest(GPUDevicePluginDSYAML)
+	ExpectNoError(err)
+	p := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "device-plugin-nvidia-gpu-" + string(uuid.NewUUID()),
+			Namespace: metav1.NamespaceSystem,
+		},
+
+		Spec: ds.Spec.Template.Spec,
+	}
+	// Remove node affinity
+	p.Spec.Affinity = nil
+
+	return p
+}
+
+func GetGPUDevicePluginImage() string {
+	ds, err := DsFromManifest(GPUDevicePluginDSYAML)
+	if err != nil {
+		klog.Errorf("Failed to parse the device plugin image: %v", err)
+		return ""
+	}
+	if ds == nil {
+		klog.Errorf("Failed to parse the device plugin image: the extracted DaemonSet is nil")
+		return ""
+	}
+	if len(ds.Spec.Template.Spec.Containers) < 1 {
+		klog.Errorf("Failed to parse the device plugin image: cannot extract the container from YAML")
+		return ""
+	}
+	return ds.Spec.Template.Spec.Containers[0].Image
+}

vendor/k8s.io/kubernetes/test/e2e/framework/jobs_util.go

@@ -0,0 +1,316 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"time"
+
+	batch "k8s.io/api/batch/v1"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	jobutil "k8s.io/kubernetes/pkg/controller/job"
+)
+
+const (
+	// How long to wait for a job to finish.
+	JobTimeout = 15 * time.Minute
+
+	// Job selector name
+	JobSelectorKey = "job"
+)
+
+// NewTestJob returns a Job which does one of several testing behaviors. notTerminate starts a Job that will run
+// effectively forever. fail starts a Job that will fail immediately. succeed starts a Job that will succeed
+// immediately. randomlySucceedOrFail starts a Job that will succeed or fail randomly. failOnce fails the Job the
+// first time it is run and succeeds subsequently. name is the Name of the Job. RestartPolicy indicates the restart
+// policy of the containers in which the Pod is running. Parallelism is the Job's parallelism, and completions is the
+// Job's required number of completions.
+func NewTestJob(behavior, name string, rPol v1.RestartPolicy, parallelism, completions int32, activeDeadlineSeconds *int64, backoffLimit int32) *batch.Job {
+	job := &batch.Job{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		TypeMeta: metav1.TypeMeta{
+			Kind: "Job",
+		},
+		Spec: batch.JobSpec{
+			ActiveDeadlineSeconds: activeDeadlineSeconds,
+			Parallelism:           &parallelism,
+			Completions:           &completions,
+			BackoffLimit:          &backoffLimit,
+			ManualSelector:        newBool(false),
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{JobSelectorKey: name},
+				},
+				Spec: v1.PodSpec{
+					RestartPolicy: rPol,
+					Volumes: []v1.Volume{
+						{
+							Name: "data",
+							VolumeSource: v1.VolumeSource{
+								EmptyDir: &v1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+					Containers: []v1.Container{
+						{
+							Name:    "c",
+							Image:   BusyBoxImage,
+							Command: []string{},
+							VolumeMounts: []v1.VolumeMount{
+								{
+									MountPath: "/data",
+									Name:      "data",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	switch behavior {
+	case "notTerminate":
+		job.Spec.Template.Spec.Containers[0].Command = []string{"sleep", "1000000"}
+	case "fail":
+		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 1"}
+	case "succeed":
+		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit 0"}
+	case "randomlySucceedOrFail":
+		// Bash's $RANDOM generates pseudorandom int in range 0 - 32767.
+		// Dividing by 16384 gives roughly 50/50 chance of success.
+		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "exit $(( $RANDOM / 16384 ))"}
+	case "failOnce":
+		// Fail the first the container of the pod is run, and
+		// succeed the second time. Checks for file on emptydir.
+		// If present, succeed.  If not, create but fail.
+		// Note that this cannot be used with RestartNever because
+		// it always fails the first time for a pod.
+		job.Spec.Template.Spec.Containers[0].Command = []string{"/bin/sh", "-c", "if [[ -r /data/foo ]] ; then exit 0 ; else touch /data/foo ; exit 1 ; fi"}
+	}
+	return job
+}
+
+// GetJob uses c to get the Job in namespace ns named name. If the returned error is nil, the returned Job is valid.
+func GetJob(c clientset.Interface, ns, name string) (*batch.Job, error) {
+	return c.BatchV1().Jobs(ns).Get(name, metav1.GetOptions{})
+}
+
+// CreateJob uses c to create job in namespace ns. If the returned error is nil, the returned Job is valid and has
+// been created.
+func CreateJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
+	return c.BatchV1().Jobs(ns).Create(job)
+}
+
+// UpdateJob uses c to updated job in namespace ns. If the returned error is nil, the returned Job is valid and has
+// been updated.
+func UpdateJob(c clientset.Interface, ns string, job *batch.Job) (*batch.Job, error) {
+	return c.BatchV1().Jobs(ns).Update(job)
+}
+
+// UpdateJobFunc updates the job object. It retries if there is a conflict, throw out error if
+// there is any other errors. name is the job name, updateFn is the function updating the
+// job object.
+func UpdateJobFunc(c clientset.Interface, ns, name string, updateFn func(job *batch.Job)) {
+	ExpectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
+		job, err := GetJob(c, ns, name)
+		if err != nil {
+			return false, fmt.Errorf("failed to get pod %q: %v", name, err)
+		}
+		updateFn(job)
+		_, err = UpdateJob(c, ns, job)
+		if err == nil {
+			Logf("Successfully updated job %q", name)
+			return true, nil
+		}
+		if errors.IsConflict(err) {
+			Logf("Conflicting update to job %q, re-get and re-update: %v", name, err)
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to update job %q: %v", name, err)
+	}))
+}
+
+// DeleteJob uses c to delete the Job named name in namespace ns. If the returned error is nil, the Job has been
+// deleted.
+func DeleteJob(c clientset.Interface, ns, name string) error {
+	return c.BatchV1().Jobs(ns).Delete(name, nil)
+}
+
+// GetJobPods returns a list of Pods belonging to a Job.
+func GetJobPods(c clientset.Interface, ns, jobName string) (*v1.PodList, error) {
+	label := labels.SelectorFromSet(labels.Set(map[string]string{JobSelectorKey: jobName}))
+	options := metav1.ListOptions{LabelSelector: label.String()}
+	return c.CoreV1().Pods(ns).List(options)
+}
+
+// WaitForAllJobPodsRunning wait for all pods for the Job named JobName in namespace ns to become Running.  Only use
+// when pods will run for a long time, or it will be racy.
+func WaitForAllJobPodsRunning(c clientset.Interface, ns, jobName string, parallelism int32) error {
+	return wait.Poll(Poll, JobTimeout, func() (bool, error) {
+		pods, err := GetJobPods(c, ns, jobName)
+		if err != nil {
+			return false, err
+		}
+		count := int32(0)
+		for _, p := range pods.Items {
+			if p.Status.Phase == v1.PodRunning {
+				count++
+			}
+		}
+		return count == parallelism, nil
+	})
+}
+
+// WaitForJobComplete uses c to wait for completions to complete for the Job jobName in namespace ns.
+func WaitForJobComplete(c clientset.Interface, ns, jobName string, completions int32) error {
+	return wait.Poll(Poll, JobTimeout, func() (bool, error) {
+		curr, err := c.BatchV1().Jobs(ns).Get(jobName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return curr.Status.Succeeded == completions, nil
+	})
+}
+
+// WaitForJobFinish uses c to wait for the Job jobName in namespace ns to finish (either Failed or Complete).
+func WaitForJobFinish(c clientset.Interface, ns, jobName string) error {
+	return wait.PollImmediate(Poll, JobTimeout, func() (bool, error) {
+		curr, err := c.BatchV1().Jobs(ns).Get(jobName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return jobutil.IsJobFinished(curr), nil
+	})
+}
+
+// WaitForJobFailure uses c to wait for up to timeout for the Job named jobName in namespace ns to fail.
+func WaitForJobFailure(c clientset.Interface, ns, jobName string, timeout time.Duration, reason string) error {
+	return wait.Poll(Poll, timeout, func() (bool, error) {
+		curr, err := c.BatchV1().Jobs(ns).Get(jobName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		for _, c := range curr.Status.Conditions {
+			if c.Type == batch.JobFailed && c.Status == v1.ConditionTrue {
+				if reason == "" || reason == c.Reason {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	})
+}
+
+// WaitForJobGone uses c to wait for up to timeout for the Job named jobName in namespace ns to be removed.
+func WaitForJobGone(c clientset.Interface, ns, jobName string, timeout time.Duration) error {
+	return wait.Poll(Poll, timeout, func() (bool, error) {
+		_, err := c.BatchV1().Jobs(ns).Get(jobName, metav1.GetOptions{})
+		if errors.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+}
+
+// CheckForAllJobPodsRunning uses c to check in the Job named jobName in ns is running. If the returned error is not
+// nil the returned bool is true if the Job is running.
+func CheckForAllJobPodsRunning(c clientset.Interface, ns, jobName string, parallelism int32) (bool, error) {
+	label := labels.SelectorFromSet(labels.Set(map[string]string{JobSelectorKey: jobName}))
+	options := metav1.ListOptions{LabelSelector: label.String()}
+	pods, err := c.CoreV1().Pods(ns).List(options)
+	if err != nil {
+		return false, err
+	}
+	count := int32(0)
+	for _, p := range pods.Items {
+		if p.Status.Phase == v1.PodRunning {
+			count++
+		}
+	}
+	return count == parallelism, nil
+}
+
+// WaitForAllJobPodsRunning wait for all pods for the Job named jobName in namespace ns
+// to be deleted.
+func WaitForAllJobPodsGone(c clientset.Interface, ns, jobName string) error {
+	return wait.PollImmediate(Poll, JobTimeout, func() (bool, error) {
+		pods, err := GetJobPods(c, ns, jobName)
+		if err != nil {
+			return false, err
+		}
+		return len(pods.Items) == 0, nil
+	})
+}
+
+func newBool(val bool) *bool {
+	p := new(bool)
+	*p = val
+	return p
+}
+
+type updateJobFunc func(*batch.Job)
+
+func UpdateJobWithRetries(c clientset.Interface, namespace, name string, applyUpdate updateJobFunc) (job *batch.Job, err error) {
+	jobs := c.BatchV1().Jobs(namespace)
+	var updateErr error
+	pollErr := wait.PollImmediate(Poll, JobTimeout, func() (bool, error) {
+		if job, err = jobs.Get(name, metav1.GetOptions{}); err != nil {
+			return false, err
+		}
+		// Apply the update, then attempt to push it to the apiserver.
+		applyUpdate(job)
+		if job, err = jobs.Update(job); err == nil {
+			Logf("Updating job %s", name)
+			return true, nil
+		}
+		updateErr = err
+		return false, nil
+	})
+	if pollErr == wait.ErrWaitTimeout {
+		pollErr = fmt.Errorf("couldn't apply the provided updated to job %q: %v", name, updateErr)
+	}
+	return job, pollErr
+}
+
+// WaitForJobDeleting uses c to wait for the Job jobName in namespace ns to have
+// a non-nil deletionTimestamp (i.e. being deleted).
+func WaitForJobDeleting(c clientset.Interface, ns, jobName string) error {
+	return wait.PollImmediate(Poll, JobTimeout, func() (bool, error) {
+		curr, err := c.BatchV1().Jobs(ns).Get(jobName, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return curr.ObjectMeta.DeletionTimestamp != nil, nil
+	})
+}
+
+func JobFinishTime(finishedJob *batch.Job) metav1.Time {
+	var finishTime metav1.Time
+	for _, c := range finishedJob.Status.Conditions {
+		if (c.Type == batch.JobComplete || c.Type == batch.JobFailed) && c.Status == v1.ConditionTrue {
+			return c.LastTransitionTime
+		}
+	}
+	return finishTime
+}

vendor/k8s.io/kubernetes/test/e2e/framework/kubelet_stats.go

@@ -0,0 +1,855 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	clientset "k8s.io/client-go/kubernetes"
+	stats "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
+	dockermetrics "k8s.io/kubernetes/pkg/kubelet/dockershim/metrics"
+	kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/test/e2e/framework/metrics"
+
+	"github.com/prometheus/common/model"
+)
+
+// KubeletMetric stores metrics scraped from the kubelet server's /metric endpoint.
+// TODO: Get some more structure around the metrics and this type
+type KubeletLatencyMetric struct {
+	// eg: list, info, create
+	Operation string
+	// eg: sync_pods, pod_worker
+	Method string
+	// 0 <= quantile <=1, e.g. 0.95 is 95%tile, 0.5 is median.
+	Quantile float64
+	Latency  time.Duration
+}
+
+// KubeletMetricByLatency implements sort.Interface for []KubeletMetric based on
+// the latency field.
+type KubeletLatencyMetrics []KubeletLatencyMetric
+
+func (a KubeletLatencyMetrics) Len() int           { return len(a) }
+func (a KubeletLatencyMetrics) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a KubeletLatencyMetrics) Less(i, j int) bool { return a[i].Latency > a[j].Latency }
+
+// If a apiserver client is passed in, the function will try to get kubelet metrics from metrics grabber;
+// or else, the function will try to get kubelet metrics directly from the node.
+func getKubeletMetricsFromNode(c clientset.Interface, nodeName string) (metrics.KubeletMetrics, error) {
+	if c == nil {
+		return metrics.GrabKubeletMetricsWithoutProxy(nodeName, "/metrics")
+	}
+	grabber, err := metrics.NewMetricsGrabber(c, nil, true, false, false, false, false)
+	if err != nil {
+		return metrics.KubeletMetrics{}, err
+	}
+	return grabber.GrabFromKubelet(nodeName)
+}
+
+// getKubeletMetrics gets all metrics in kubelet subsystem from specified node and trims
+// the subsystem prefix.
+func getKubeletMetrics(c clientset.Interface, nodeName string) (metrics.KubeletMetrics, error) {
+	ms, err := getKubeletMetricsFromNode(c, nodeName)
+	if err != nil {
+		return metrics.KubeletMetrics{}, err
+	}
+
+	kubeletMetrics := make(metrics.KubeletMetrics)
+	for name, samples := range ms {
+		const prefix = kubeletmetrics.KubeletSubsystem + "_"
+		if !strings.HasPrefix(name, prefix) {
+			// Not a kubelet metric.
+			continue
+		}
+		method := strings.TrimPrefix(name, prefix)
+		kubeletMetrics[method] = samples
+	}
+	return kubeletMetrics, nil
+}
+
+// GetDefaultKubeletLatencyMetrics calls GetKubeletLatencyMetrics with a set of default metricNames
+// identifying common latency metrics.
+// Note that the KubeletMetrics passed in should not contain subsystem prefix.
+func GetDefaultKubeletLatencyMetrics(ms metrics.KubeletMetrics) KubeletLatencyMetrics {
+	latencyMetricNames := sets.NewString(
+		kubeletmetrics.PodWorkerDurationKey,
+		kubeletmetrics.PodWorkerStartDurationKey,
+		kubeletmetrics.PodStartDurationKey,
+		kubeletmetrics.CgroupManagerOperationsKey,
+		dockermetrics.DockerOperationsLatencyKey,
+		kubeletmetrics.PodWorkerStartDurationKey,
+		kubeletmetrics.PLEGRelistDurationKey,
+	)
+	return GetKubeletLatencyMetrics(ms, latencyMetricNames)
+}
+
+// GetKubeletLatencyMetrics filters ms to include only those contained in the metricNames set,
+// then constructs a KubeletLatencyMetrics list based on the samples associated with those metrics.
+func GetKubeletLatencyMetrics(ms metrics.KubeletMetrics, filterMetricNames sets.String) KubeletLatencyMetrics {
+	var latencyMetrics KubeletLatencyMetrics
+	for name, samples := range ms {
+		if !filterMetricNames.Has(name) {
+			continue
+		}
+		for _, sample := range samples {
+			latency := sample.Value
+			operation := string(sample.Metric["operation_type"])
+			var quantile float64
+			if val, ok := sample.Metric[model.QuantileLabel]; ok {
+				var err error
+				if quantile, err = strconv.ParseFloat(string(val), 64); err != nil {
+					continue
+				}
+			}
+
+			latencyMetrics = append(latencyMetrics, KubeletLatencyMetric{
+				Operation: operation,
+				Method:    name,
+				Quantile:  quantile,
+				Latency:   time.Duration(int64(latency)) * time.Microsecond,
+			})
+		}
+	}
+	return latencyMetrics
+}
+
+// RuntimeOperationMonitor is the tool getting and parsing docker operation metrics.
+type RuntimeOperationMonitor struct {
+	client          clientset.Interface
+	nodesRuntimeOps map[string]NodeRuntimeOperationErrorRate
+}
+
+// NodeRuntimeOperationErrorRate is the runtime operation error rate on one node.
+type NodeRuntimeOperationErrorRate map[string]*RuntimeOperationErrorRate
+
+// RuntimeOperationErrorRate is the error rate of a specified runtime operation.
+type RuntimeOperationErrorRate struct {
+	TotalNumber float64
+	ErrorRate   float64
+	TimeoutRate float64
+}
+
+func NewRuntimeOperationMonitor(c clientset.Interface) *RuntimeOperationMonitor {
+	m := &RuntimeOperationMonitor{
+		client:          c,
+		nodesRuntimeOps: make(map[string]NodeRuntimeOperationErrorRate),
+	}
+	nodes, err := m.client.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		Failf("RuntimeOperationMonitor: unable to get list of nodes: %v", err)
+	}
+	for _, node := range nodes.Items {
+		m.nodesRuntimeOps[node.Name] = make(NodeRuntimeOperationErrorRate)
+	}
+	// Initialize the runtime operation error rate
+	m.GetRuntimeOperationErrorRate()
+	return m
+}
+
+// GetRuntimeOperationErrorRate gets runtime operation records from kubelet metrics and calculate
+// error rates of all runtime operations.
+func (m *RuntimeOperationMonitor) GetRuntimeOperationErrorRate() map[string]NodeRuntimeOperationErrorRate {
+	for node := range m.nodesRuntimeOps {
+		nodeResult, err := getNodeRuntimeOperationErrorRate(m.client, node)
+		if err != nil {
+			Logf("GetRuntimeOperationErrorRate: unable to get kubelet metrics from node %q: %v", node, err)
+			continue
+		}
+		m.nodesRuntimeOps[node] = nodeResult
+	}
+	return m.nodesRuntimeOps
+}
+
+// GetLatestRuntimeOperationErrorRate gets latest error rate and timeout rate from last observed RuntimeOperationErrorRate.
+func (m *RuntimeOperationMonitor) GetLatestRuntimeOperationErrorRate() map[string]NodeRuntimeOperationErrorRate {
+	result := make(map[string]NodeRuntimeOperationErrorRate)
+	for node := range m.nodesRuntimeOps {
+		result[node] = make(NodeRuntimeOperationErrorRate)
+		oldNodeResult := m.nodesRuntimeOps[node]
+		curNodeResult, err := getNodeRuntimeOperationErrorRate(m.client, node)
+		if err != nil {
+			Logf("GetLatestRuntimeOperationErrorRate: unable to get kubelet metrics from node %q: %v", node, err)
+			continue
+		}
+		for op, cur := range curNodeResult {
+			t := *cur
+			if old, found := oldNodeResult[op]; found {
+				t.ErrorRate = (t.ErrorRate*t.TotalNumber - old.ErrorRate*old.TotalNumber) / (t.TotalNumber - old.TotalNumber)
+				t.TimeoutRate = (t.TimeoutRate*t.TotalNumber - old.TimeoutRate*old.TotalNumber) / (t.TotalNumber - old.TotalNumber)
+				t.TotalNumber -= old.TotalNumber
+			}
+			result[node][op] = &t
+		}
+		m.nodesRuntimeOps[node] = curNodeResult
+	}
+	return result
+}
+
+// FormatRuntimeOperationErrorRate formats the runtime operation error rate to string.
+func FormatRuntimeOperationErrorRate(nodesResult map[string]NodeRuntimeOperationErrorRate) string {
+	lines := []string{}
+	for node, nodeResult := range nodesResult {
+		lines = append(lines, fmt.Sprintf("node %q runtime operation error rate:", node))
+		for op, result := range nodeResult {
+			line := fmt.Sprintf("operation %q: total - %.0f; error rate - %f; timeout rate - %f", op,
+				result.TotalNumber, result.ErrorRate, result.TimeoutRate)
+			lines = append(lines, line)
+		}
+		lines = append(lines, fmt.Sprintln())
+	}
+	return strings.Join(lines, "\n")
+}
+
+// getNodeRuntimeOperationErrorRate gets runtime operation error rate from specified node.
+func getNodeRuntimeOperationErrorRate(c clientset.Interface, node string) (NodeRuntimeOperationErrorRate, error) {
+	result := make(NodeRuntimeOperationErrorRate)
+	ms, err := getKubeletMetrics(c, node)
+	if err != nil {
+		return result, err
+	}
+	// If no corresponding metrics are found, the returned samples will be empty. Then the following
+	// loop will be skipped automatically.
+	allOps := ms[dockermetrics.DockerOperationsKey]
+	errOps := ms[dockermetrics.DockerOperationsErrorsKey]
+	timeoutOps := ms[dockermetrics.DockerOperationsTimeoutKey]
+	for _, sample := range allOps {
+		operation := string(sample.Metric["operation_type"])
+		result[operation] = &RuntimeOperationErrorRate{TotalNumber: float64(sample.Value)}
+	}
+	for _, sample := range errOps {
+		operation := string(sample.Metric["operation_type"])
+		// Should always find the corresponding item, just in case
+		if _, found := result[operation]; found {
+			result[operation].ErrorRate = float64(sample.Value) / result[operation].TotalNumber
+		}
+	}
+	for _, sample := range timeoutOps {
+		operation := string(sample.Metric["operation_type"])
+		if _, found := result[operation]; found {
+			result[operation].TimeoutRate = float64(sample.Value) / result[operation].TotalNumber
+		}
+	}
+	return result, nil
+}
+
+// HighLatencyKubeletOperations logs and counts the high latency metrics exported by the kubelet server via /metrics.
+func HighLatencyKubeletOperations(c clientset.Interface, threshold time.Duration, nodeName string, logFunc func(fmt string, args ...interface{})) (KubeletLatencyMetrics, error) {
+	ms, err := getKubeletMetrics(c, nodeName)
+	if err != nil {
+		return KubeletLatencyMetrics{}, err
+	}
+	latencyMetrics := GetDefaultKubeletLatencyMetrics(ms)
+	sort.Sort(latencyMetrics)
+	var badMetrics KubeletLatencyMetrics
+	logFunc("\nLatency metrics for node %v", nodeName)
+	for _, m := range latencyMetrics {
+		if m.Latency > threshold {
+			badMetrics = append(badMetrics, m)
+			Logf("%+v", m)
+		}
+	}
+	return badMetrics, nil
+}
+
+// GetStatsSummary contacts kubelet for the container information.
+func GetStatsSummary(c clientset.Interface, nodeName string) (*stats.Summary, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout)
+	defer cancel()
+
+	data, err := c.CoreV1().RESTClient().Get().
+		Context(ctx).
+		Resource("nodes").
+		SubResource("proxy").
+		Name(fmt.Sprintf("%v:%v", nodeName, ports.KubeletPort)).
+		Suffix("stats/summary").
+		Do().Raw()
+
+	if err != nil {
+		return nil, err
+	}
+
+	summary := stats.Summary{}
+	err = json.Unmarshal(data, &summary)
+	if err != nil {
+		return nil, err
+	}
+	return &summary, nil
+}
+
+func removeUint64Ptr(ptr *uint64) uint64 {
+	if ptr == nil {
+		return 0
+	}
+	return *ptr
+}
+
+// getOneTimeResourceUsageOnNode queries the node's /stats/summary endpoint
+// and returns the resource usage of all containerNames for the past
+// cpuInterval.
+// The acceptable range of the interval is 2s~120s. Be warned that as the
+// interval (and #containers) increases, the size of kubelet's response
+// could be significant. E.g., the 60s interval stats for ~20 containers is
+// ~1.5MB. Don't hammer the node with frequent, heavy requests.
+//
+// cadvisor records cumulative cpu usage in nanoseconds, so we need to have two
+// stats points to compute the cpu usage over the interval. Assuming cadvisor
+// polls every second, we'd need to get N stats points for N-second interval.
+// Note that this is an approximation and may not be accurate, hence we also
+// write the actual interval used for calculation (based on the timestamps of
+// the stats points in ContainerResourceUsage.CPUInterval.
+//
+// containerNames is a function returning a collection of container names in which
+// user is interested in.
+func getOneTimeResourceUsageOnNode(
+	c clientset.Interface,
+	nodeName string,
+	cpuInterval time.Duration,
+	containerNames func() []string,
+) (ResourceUsagePerContainer, error) {
+	const (
+		// cadvisor records stats about every second.
+		cadvisorStatsPollingIntervalInSeconds float64 = 1.0
+		// cadvisor caches up to 2 minutes of stats (configured by kubelet).
+		maxNumStatsToRequest int = 120
+	)
+
+	numStats := int(float64(cpuInterval.Seconds()) / cadvisorStatsPollingIntervalInSeconds)
+	if numStats < 2 || numStats > maxNumStatsToRequest {
+		return nil, fmt.Errorf("numStats needs to be > 1 and < %d", maxNumStatsToRequest)
+	}
+	// Get information of all containers on the node.
+	summary, err := GetStatsSummary(c, nodeName)
+	if err != nil {
+		return nil, err
+	}
+
+	f := func(name string, newStats *stats.ContainerStats) *ContainerResourceUsage {
+		if newStats == nil || newStats.CPU == nil || newStats.Memory == nil {
+			return nil
+		}
+		return &ContainerResourceUsage{
+			Name:                    name,
+			Timestamp:               newStats.StartTime.Time,
+			CPUUsageInCores:         float64(removeUint64Ptr(newStats.CPU.UsageNanoCores)) / 1000000000,
+			MemoryUsageInBytes:      removeUint64Ptr(newStats.Memory.UsageBytes),
+			MemoryWorkingSetInBytes: removeUint64Ptr(newStats.Memory.WorkingSetBytes),
+			MemoryRSSInBytes:        removeUint64Ptr(newStats.Memory.RSSBytes),
+			CPUInterval:             0,
+		}
+	}
+	// Process container infos that are relevant to us.
+	containers := containerNames()
+	usageMap := make(ResourceUsagePerContainer, len(containers))
+	observedContainers := []string{}
+	for _, pod := range summary.Pods {
+		for _, container := range pod.Containers {
+			isInteresting := false
+			for _, interestingContainerName := range containers {
+				if container.Name == interestingContainerName {
+					isInteresting = true
+					observedContainers = append(observedContainers, container.Name)
+					break
+				}
+			}
+			if !isInteresting {
+				continue
+			}
+			if usage := f(pod.PodRef.Name+"/"+container.Name, &container); usage != nil {
+				usageMap[pod.PodRef.Name+"/"+container.Name] = usage
+			}
+		}
+	}
+	return usageMap, nil
+}
+
+func getNodeStatsSummary(c clientset.Interface, nodeName string) (*stats.Summary, error) {
+	data, err := c.CoreV1().RESTClient().Get().
+		Resource("nodes").
+		SubResource("proxy").
+		Name(fmt.Sprintf("%v:%v", nodeName, ports.KubeletPort)).
+		Suffix("stats/summary").
+		SetHeader("Content-Type", "application/json").
+		Do().Raw()
+
+	if err != nil {
+		return nil, err
+	}
+
+	var summary *stats.Summary
+	err = json.Unmarshal(data, &summary)
+	if err != nil {
+		return nil, err
+	}
+	return summary, nil
+}
+
+func getSystemContainerStats(summary *stats.Summary) map[string]*stats.ContainerStats {
+	statsList := summary.Node.SystemContainers
+	statsMap := make(map[string]*stats.ContainerStats)
+	for i := range statsList {
+		statsMap[statsList[i].Name] = &statsList[i]
+	}
+
+	// Create a root container stats using information available in
+	// stats.NodeStats. This is necessary since it is a different type.
+	statsMap[rootContainerName] = &stats.ContainerStats{
+		CPU:    summary.Node.CPU,
+		Memory: summary.Node.Memory,
+	}
+	return statsMap
+}
+
+const (
+	rootContainerName = "/"
+)
+
+// A list of containers for which we want to collect resource usage.
+func TargetContainers() []string {
+	return []string{
+		rootContainerName,
+		stats.SystemContainerRuntime,
+		stats.SystemContainerKubelet,
+	}
+}
+
+type ContainerResourceUsage struct {
+	Name                    string
+	Timestamp               time.Time
+	CPUUsageInCores         float64
+	MemoryUsageInBytes      uint64
+	MemoryWorkingSetInBytes uint64
+	MemoryRSSInBytes        uint64
+	// The interval used to calculate CPUUsageInCores.
+	CPUInterval time.Duration
+}
+
+func (r *ContainerResourceUsage) isStrictlyGreaterThan(rhs *ContainerResourceUsage) bool {
+	return r.CPUUsageInCores > rhs.CPUUsageInCores && r.MemoryWorkingSetInBytes > rhs.MemoryWorkingSetInBytes
+}
+
+type ResourceUsagePerContainer map[string]*ContainerResourceUsage
+type ResourceUsagePerNode map[string]ResourceUsagePerContainer
+
+func formatResourceUsageStats(nodeName string, containerStats ResourceUsagePerContainer) string {
+	// Example output:
+	//
+	// Resource usage for node "e2e-test-foo-node-abcde":
+	// container        cpu(cores)  memory(MB)
+	// "/"              0.363       2942.09
+	// "/docker-daemon" 0.088       521.80
+	// "/kubelet"       0.086       424.37
+	// "/system"        0.007       119.88
+	buf := &bytes.Buffer{}
+	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "container\tcpu(cores)\tmemory_working_set(MB)\tmemory_rss(MB)\n")
+	for name, s := range containerStats {
+		fmt.Fprintf(w, "%q\t%.3f\t%.2f\t%.2f\n", name, s.CPUUsageInCores, float64(s.MemoryWorkingSetInBytes)/(1024*1024), float64(s.MemoryRSSInBytes)/(1024*1024))
+	}
+	w.Flush()
+	return fmt.Sprintf("Resource usage on node %q:\n%s", nodeName, buf.String())
+}
+
+type uint64arr []uint64
+
+func (a uint64arr) Len() int           { return len(a) }
+func (a uint64arr) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a uint64arr) Less(i, j int) bool { return a[i] < a[j] }
+
+type usageDataPerContainer struct {
+	cpuData        []float64
+	memUseData     []uint64
+	memWorkSetData []uint64
+}
+
+func GetKubeletHeapStats(c clientset.Interface, nodeName string) (string, error) {
+	client, err := NodeProxyRequest(c, nodeName, "debug/pprof/heap", ports.KubeletPort)
+	if err != nil {
+		return "", err
+	}
+	raw, errRaw := client.Raw()
+	if errRaw != nil {
+		return "", err
+	}
+	stats := string(raw)
+	// Only dumping the runtime.MemStats numbers to avoid polluting the log.
+	numLines := 23
+	lines := strings.Split(stats, "\n")
+	return strings.Join(lines[len(lines)-numLines:], "\n"), nil
+}
+
+func PrintAllKubeletPods(c clientset.Interface, nodeName string) {
+	podList, err := GetKubeletPods(c, nodeName)
+	if err != nil {
+		Logf("Unable to retrieve kubelet pods for node %v: %v", nodeName, err)
+		return
+	}
+	for _, p := range podList.Items {
+		Logf("%v from %v started at %v (%d container statuses recorded)", p.Name, p.Namespace, p.Status.StartTime, len(p.Status.ContainerStatuses))
+		for _, c := range p.Status.ContainerStatuses {
+			Logf("\tContainer %v ready: %v, restart count %v",
+				c.Name, c.Ready, c.RestartCount)
+		}
+	}
+}
+
+func computeContainerResourceUsage(name string, oldStats, newStats *stats.ContainerStats) *ContainerResourceUsage {
+	return &ContainerResourceUsage{
+		Name:                    name,
+		Timestamp:               newStats.CPU.Time.Time,
+		CPUUsageInCores:         float64(*newStats.CPU.UsageCoreNanoSeconds-*oldStats.CPU.UsageCoreNanoSeconds) / float64(newStats.CPU.Time.Time.Sub(oldStats.CPU.Time.Time).Nanoseconds()),
+		MemoryUsageInBytes:      *newStats.Memory.UsageBytes,
+		MemoryWorkingSetInBytes: *newStats.Memory.WorkingSetBytes,
+		MemoryRSSInBytes:        *newStats.Memory.RSSBytes,
+		CPUInterval:             newStats.CPU.Time.Time.Sub(oldStats.CPU.Time.Time),
+	}
+}
+
+// resourceCollector periodically polls the node, collect stats for a given
+// list of containers, computes and cache resource usage up to
+// maxEntriesPerContainer for each container.
+type resourceCollector struct {
+	lock            sync.RWMutex
+	node            string
+	containers      []string
+	client          clientset.Interface
+	buffers         map[string][]*ContainerResourceUsage
+	pollingInterval time.Duration
+	stopCh          chan struct{}
+}
+
+func newResourceCollector(c clientset.Interface, nodeName string, containerNames []string, pollingInterval time.Duration) *resourceCollector {
+	buffers := make(map[string][]*ContainerResourceUsage)
+	return &resourceCollector{
+		node:            nodeName,
+		containers:      containerNames,
+		client:          c,
+		buffers:         buffers,
+		pollingInterval: pollingInterval,
+	}
+}
+
+// Start starts a goroutine to Poll the node every pollingInterval.
+func (r *resourceCollector) Start() {
+	r.stopCh = make(chan struct{}, 1)
+	// Keep the last observed stats for comparison.
+	oldStats := make(map[string]*stats.ContainerStats)
+	go wait.Until(func() { r.collectStats(oldStats) }, r.pollingInterval, r.stopCh)
+}
+
+// Stop sends a signal to terminate the stats collecting goroutine.
+func (r *resourceCollector) Stop() {
+	close(r.stopCh)
+}
+
+// collectStats gets the latest stats from kubelet stats summary API, computes
+// the resource usage, and pushes it to the buffer.
+func (r *resourceCollector) collectStats(oldStatsMap map[string]*stats.ContainerStats) {
+	summary, err := getNodeStatsSummary(r.client, r.node)
+	if err != nil {
+		Logf("Error getting node stats summary on %q, err: %v", r.node, err)
+		return
+	}
+	cStatsMap := getSystemContainerStats(summary)
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	for _, name := range r.containers {
+		cStats, ok := cStatsMap[name]
+		if !ok {
+			Logf("Missing info/stats for container %q on node %q", name, r.node)
+			return
+		}
+
+		if oldStats, ok := oldStatsMap[name]; ok {
+			if oldStats.CPU.Time.Equal(&cStats.CPU.Time) {
+				// No change -> skip this stat.
+				continue
+			}
+			r.buffers[name] = append(r.buffers[name], computeContainerResourceUsage(name, oldStats, cStats))
+		}
+		// Update the old stats.
+		oldStatsMap[name] = cStats
+	}
+}
+
+func (r *resourceCollector) GetLatest() (ResourceUsagePerContainer, error) {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+	stats := make(ResourceUsagePerContainer)
+	for _, name := range r.containers {
+		contStats, ok := r.buffers[name]
+		if !ok || len(contStats) == 0 {
+			return nil, fmt.Errorf("Resource usage on node %q is not ready yet", r.node)
+		}
+		stats[name] = contStats[len(contStats)-1]
+	}
+	return stats, nil
+}
+
+// Reset frees the stats and start over.
+func (r *resourceCollector) Reset() {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	for _, name := range r.containers {
+		r.buffers[name] = []*ContainerResourceUsage{}
+	}
+}
+
+type resourceUsageByCPU []*ContainerResourceUsage
+
+func (r resourceUsageByCPU) Len() int           { return len(r) }
+func (r resourceUsageByCPU) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+func (r resourceUsageByCPU) Less(i, j int) bool { return r[i].CPUUsageInCores < r[j].CPUUsageInCores }
+
+// The percentiles to report.
+var percentiles = [...]float64{0.05, 0.20, 0.50, 0.70, 0.90, 0.95, 0.99}
+
+// GetBasicCPUStats returns the percentiles the cpu usage in cores for
+// containerName. This method examines all data currently in the buffer.
+func (r *resourceCollector) GetBasicCPUStats(containerName string) map[float64]float64 {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+	result := make(map[float64]float64, len(percentiles))
+	usages := r.buffers[containerName]
+	sort.Sort(resourceUsageByCPU(usages))
+	for _, q := range percentiles {
+		index := int(float64(len(usages))*q) - 1
+		if index < 0 {
+			// We don't have enough data.
+			result[q] = 0
+			continue
+		}
+		result[q] = usages[index].CPUUsageInCores
+	}
+	return result
+}
+
+// ResourceMonitor manages a resourceCollector per node.
+type ResourceMonitor struct {
+	client          clientset.Interface
+	containers      []string
+	pollingInterval time.Duration
+	collectors      map[string]*resourceCollector
+}
+
+func NewResourceMonitor(c clientset.Interface, containerNames []string, pollingInterval time.Duration) *ResourceMonitor {
+	return &ResourceMonitor{
+		containers:      containerNames,
+		client:          c,
+		pollingInterval: pollingInterval,
+	}
+}
+
+func (r *ResourceMonitor) Start() {
+	// It should be OK to monitor unschedulable Nodes
+	nodes, err := r.client.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		Failf("ResourceMonitor: unable to get list of nodes: %v", err)
+	}
+	r.collectors = make(map[string]*resourceCollector, 0)
+	for _, node := range nodes.Items {
+		collector := newResourceCollector(r.client, node.Name, r.containers, r.pollingInterval)
+		r.collectors[node.Name] = collector
+		collector.Start()
+	}
+}
+
+func (r *ResourceMonitor) Stop() {
+	for _, collector := range r.collectors {
+		collector.Stop()
+	}
+}
+
+func (r *ResourceMonitor) Reset() {
+	for _, collector := range r.collectors {
+		collector.Reset()
+	}
+}
+
+func (r *ResourceMonitor) LogLatest() {
+	summary, err := r.GetLatest()
+	if err != nil {
+		Logf("%v", err)
+	}
+	Logf("%s", r.FormatResourceUsage(summary))
+}
+
+func (r *ResourceMonitor) FormatResourceUsage(s ResourceUsagePerNode) string {
+	summary := []string{}
+	for node, usage := range s {
+		summary = append(summary, formatResourceUsageStats(node, usage))
+	}
+	return strings.Join(summary, "\n")
+}
+
+func (r *ResourceMonitor) GetLatest() (ResourceUsagePerNode, error) {
+	result := make(ResourceUsagePerNode)
+	errs := []error{}
+	for key, collector := range r.collectors {
+		s, err := collector.GetLatest()
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		result[key] = s
+	}
+	return result, utilerrors.NewAggregate(errs)
+}
+
+func (r *ResourceMonitor) GetMasterNodeLatest(usagePerNode ResourceUsagePerNode) ResourceUsagePerNode {
+	result := make(ResourceUsagePerNode)
+	var masterUsage ResourceUsagePerContainer
+	var nodesUsage []ResourceUsagePerContainer
+	for node, usage := range usagePerNode {
+		if strings.HasSuffix(node, "master") {
+			masterUsage = usage
+		} else {
+			nodesUsage = append(nodesUsage, usage)
+		}
+	}
+	nodeAvgUsage := make(ResourceUsagePerContainer)
+	for _, nodeUsage := range nodesUsage {
+		for c, usage := range nodeUsage {
+			if _, found := nodeAvgUsage[c]; !found {
+				nodeAvgUsage[c] = &ContainerResourceUsage{Name: usage.Name}
+			}
+			nodeAvgUsage[c].CPUUsageInCores += usage.CPUUsageInCores
+			nodeAvgUsage[c].MemoryUsageInBytes += usage.MemoryUsageInBytes
+			nodeAvgUsage[c].MemoryWorkingSetInBytes += usage.MemoryWorkingSetInBytes
+			nodeAvgUsage[c].MemoryRSSInBytes += usage.MemoryRSSInBytes
+		}
+	}
+	for c := range nodeAvgUsage {
+		nodeAvgUsage[c].CPUUsageInCores /= float64(len(nodesUsage))
+		nodeAvgUsage[c].MemoryUsageInBytes /= uint64(len(nodesUsage))
+		nodeAvgUsage[c].MemoryWorkingSetInBytes /= uint64(len(nodesUsage))
+		nodeAvgUsage[c].MemoryRSSInBytes /= uint64(len(nodesUsage))
+	}
+	result["master"] = masterUsage
+	result["node"] = nodeAvgUsage
+	return result
+}
+
+// ContainersCPUSummary is indexed by the container name with each entry a
+// (percentile, value) map.
+type ContainersCPUSummary map[string]map[float64]float64
+
+// NodesCPUSummary is indexed by the node name with each entry a
+// ContainersCPUSummary map.
+type NodesCPUSummary map[string]ContainersCPUSummary
+
+func (r *ResourceMonitor) FormatCPUSummary(summary NodesCPUSummary) string {
+	// Example output for a node (the percentiles may differ):
+	// CPU usage of containers on node "e2e-test-foo-node-0vj7":
+	// container        5th%  50th% 90th% 95th%
+	// "/"              0.051 0.159 0.387 0.455
+	// "/runtime        0.000 0.000 0.146 0.166
+	// "/kubelet"       0.036 0.053 0.091 0.154
+	// "/misc"          0.001 0.001 0.001 0.002
+	var summaryStrings []string
+	var header []string
+	header = append(header, "container")
+	for _, p := range percentiles {
+		header = append(header, fmt.Sprintf("%.0fth%%", p*100))
+	}
+	for nodeName, containers := range summary {
+		buf := &bytes.Buffer{}
+		w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+		fmt.Fprintf(w, "%s\n", strings.Join(header, "\t"))
+		for _, containerName := range TargetContainers() {
+			var s []string
+			s = append(s, fmt.Sprintf("%q", containerName))
+			data, ok := containers[containerName]
+			for _, p := range percentiles {
+				value := "N/A"
+				if ok {
+					value = fmt.Sprintf("%.3f", data[p])
+				}
+				s = append(s, value)
+			}
+			fmt.Fprintf(w, "%s\n", strings.Join(s, "\t"))
+		}
+		w.Flush()
+		summaryStrings = append(summaryStrings, fmt.Sprintf("CPU usage of containers on node %q\n:%s", nodeName, buf.String()))
+	}
+	return strings.Join(summaryStrings, "\n")
+}
+
+func (r *ResourceMonitor) LogCPUSummary() {
+	summary := r.GetCPUSummary()
+	Logf("%s", r.FormatCPUSummary(summary))
+}
+
+func (r *ResourceMonitor) GetCPUSummary() NodesCPUSummary {
+	result := make(NodesCPUSummary)
+	for nodeName, collector := range r.collectors {
+		result[nodeName] = make(ContainersCPUSummary)
+		for _, containerName := range TargetContainers() {
+			data := collector.GetBasicCPUStats(containerName)
+			result[nodeName][containerName] = data
+		}
+	}
+	return result
+}
+
+func (r *ResourceMonitor) GetMasterNodeCPUSummary(summaryPerNode NodesCPUSummary) NodesCPUSummary {
+	result := make(NodesCPUSummary)
+	var masterSummary ContainersCPUSummary
+	var nodesSummaries []ContainersCPUSummary
+	for node, summary := range summaryPerNode {
+		if strings.HasSuffix(node, "master") {
+			masterSummary = summary
+		} else {
+			nodesSummaries = append(nodesSummaries, summary)
+		}
+	}
+
+	nodeAvgSummary := make(ContainersCPUSummary)
+	for _, nodeSummary := range nodesSummaries {
+		for c, summary := range nodeSummary {
+			if _, found := nodeAvgSummary[c]; !found {
+				nodeAvgSummary[c] = map[float64]float64{}
+			}
+			for perc, value := range summary {
+				nodeAvgSummary[c][perc] += value
+			}
+		}
+	}
+	for c := range nodeAvgSummary {
+		for perc := range nodeAvgSummary[c] {
+			nodeAvgSummary[c][perc] /= float64(len(nodesSummaries))
+		}
+	}
+	result["master"] = masterSummary
+	result["node"] = nodeAvgSummary
+	return result
+}

vendor/k8s.io/kubernetes/test/e2e/framework/log_size_monitoring.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+const (
+	// Minimal period between polling log sizes from components
+	pollingPeriod            = 60 * time.Second
+	workersNo                = 5
+	kubeletLogsPath          = "/var/log/kubelet.log"
+	kubeProxyLogsPath        = "/var/log/kube-proxy.log"
+	kubeAddonsLogsPath       = "/var/log/kube-addons.log"
+	kubeMasterAddonsLogsPath = "/var/log/kube-master-addons.log"
+	apiServerLogsPath        = "/var/log/kube-apiserver.log"
+	controllersLogsPath      = "/var/log/kube-controller-manager.log"
+	schedulerLogsPath        = "/var/log/kube-scheduler.log"
+)
+
+var (
+	nodeLogsToCheck   = []string{kubeletLogsPath, kubeProxyLogsPath}
+	masterLogsToCheck = []string{kubeletLogsPath, kubeAddonsLogsPath, kubeMasterAddonsLogsPath,
+		apiServerLogsPath, controllersLogsPath, schedulerLogsPath}
+)
+
+// TimestampedSize contains a size together with a time of measurement.
+type TimestampedSize struct {
+	timestamp time.Time
+	size      int
+}
+
+// LogSizeGatherer is a worker which grabs a WorkItem from the channel and does assigned work.
+type LogSizeGatherer struct {
+	stopChannel chan bool
+	data        *LogsSizeData
+	wg          *sync.WaitGroup
+	workChannel chan WorkItem
+}
+
+// LogsSizeVerifier gathers data about log files sizes from master and node machines.
+// It oversees a <workersNo> workers which do the gathering.
+type LogsSizeVerifier struct {
+	client      clientset.Interface
+	stopChannel chan bool
+	// data stores LogSizeData groupped per IP and log_path
+	data          *LogsSizeData
+	masterAddress string
+	nodeAddresses []string
+	wg            sync.WaitGroup
+	workChannel   chan WorkItem
+	workers       []*LogSizeGatherer
+}
+
+type SingleLogSummary struct {
+	AverageGenerationRate int
+	NumberOfProbes        int
+}
+
+type LogSizeDataTimeseries map[string]map[string][]TimestampedSize
+
+// node -> file -> data
+type LogsSizeDataSummary map[string]map[string]SingleLogSummary
+
+// TODO: make sure that we don't need locking here
+func (s *LogsSizeDataSummary) PrintHumanReadable() string {
+	buf := &bytes.Buffer{}
+	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "host\tlog_file\taverage_rate (B/s)\tnumber_of_probes\n")
+	for k, v := range *s {
+		fmt.Fprintf(w, "%v\t\t\t\n", k)
+		for path, data := range v {
+			fmt.Fprintf(w, "\t%v\t%v\t%v\n", path, data.AverageGenerationRate, data.NumberOfProbes)
+		}
+	}
+	w.Flush()
+	return buf.String()
+}
+
+func (s *LogsSizeDataSummary) PrintJSON() string {
+	return PrettyPrintJSON(*s)
+}
+
+func (s *LogsSizeDataSummary) SummaryKind() string {
+	return "LogSizeSummary"
+}
+
+type LogsSizeData struct {
+	data LogSizeDataTimeseries
+	lock sync.Mutex
+}
+
+// WorkItem is a command for a worker that contains an IP of machine from which we want to
+// gather data and paths to all files we're interested in.
+type WorkItem struct {
+	ip                string
+	paths             []string
+	backoffMultiplier int
+}
+
+func prepareData(masterAddress string, nodeAddresses []string) *LogsSizeData {
+	data := make(LogSizeDataTimeseries)
+	ips := append(nodeAddresses, masterAddress)
+	for _, ip := range ips {
+		data[ip] = make(map[string][]TimestampedSize)
+	}
+	return &LogsSizeData{
+		data: data,
+		lock: sync.Mutex{},
+	}
+}
+
+func (d *LogsSizeData) AddNewData(ip, path string, timestamp time.Time, size int) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.data[ip][path] = append(
+		d.data[ip][path],
+		TimestampedSize{
+			timestamp: timestamp,
+			size:      size,
+		},
+	)
+}
+
+// NewLogsVerifier creates a new LogsSizeVerifier which will stop when stopChannel is closed
+func NewLogsVerifier(c clientset.Interface, stopChannel chan bool) *LogsSizeVerifier {
+	nodeAddresses, err := NodeSSHHosts(c)
+	ExpectNoError(err)
+	masterAddress := GetMasterHost() + ":22"
+
+	workChannel := make(chan WorkItem, len(nodeAddresses)+1)
+	workers := make([]*LogSizeGatherer, workersNo)
+
+	verifier := &LogsSizeVerifier{
+		client:        c,
+		stopChannel:   stopChannel,
+		data:          prepareData(masterAddress, nodeAddresses),
+		masterAddress: masterAddress,
+		nodeAddresses: nodeAddresses,
+		wg:            sync.WaitGroup{},
+		workChannel:   workChannel,
+		workers:       workers,
+	}
+	verifier.wg.Add(workersNo)
+	for i := 0; i < workersNo; i++ {
+		workers[i] = &LogSizeGatherer{
+			stopChannel: stopChannel,
+			data:        verifier.data,
+			wg:          &verifier.wg,
+			workChannel: workChannel,
+		}
+	}
+	return verifier
+}
+
+// GetSummary returns a summary (average generation rate and number of probes) of the data gathered by LogSizeVerifier
+func (s *LogsSizeVerifier) GetSummary() *LogsSizeDataSummary {
+	result := make(LogsSizeDataSummary)
+	for k, v := range s.data.data {
+		result[k] = make(map[string]SingleLogSummary)
+		for path, data := range v {
+			if len(data) > 1 {
+				last := data[len(data)-1]
+				first := data[0]
+				rate := (last.size - first.size) / int(last.timestamp.Sub(first.timestamp)/time.Second)
+				result[k][path] = SingleLogSummary{
+					AverageGenerationRate: rate,
+					NumberOfProbes:        len(data),
+				}
+			}
+		}
+	}
+	return &result
+}
+
+// Run starts log size gathering. It starts a gorouting for every worker and then blocks until stopChannel is closed
+func (v *LogsSizeVerifier) Run() {
+	v.workChannel <- WorkItem{
+		ip:                v.masterAddress,
+		paths:             masterLogsToCheck,
+		backoffMultiplier: 1,
+	}
+	for _, node := range v.nodeAddresses {
+		v.workChannel <- WorkItem{
+			ip:                node,
+			paths:             nodeLogsToCheck,
+			backoffMultiplier: 1,
+		}
+	}
+	for _, worker := range v.workers {
+		go worker.Run()
+	}
+	<-v.stopChannel
+	v.wg.Wait()
+}
+
+func (g *LogSizeGatherer) Run() {
+	for g.Work() {
+	}
+}
+
+func (g *LogSizeGatherer) pushWorkItem(workItem WorkItem) {
+	select {
+	case <-time.After(time.Duration(workItem.backoffMultiplier) * pollingPeriod):
+		g.workChannel <- workItem
+	case <-g.stopChannel:
+		return
+	}
+}
+
+// Work does a single unit of work: tries to take out a WorkItem from the queue, ssh-es into a given machine,
+// gathers data, writes it to the shared <data> map, and creates a gorouting which reinserts work item into
+// the queue with a <pollingPeriod> delay. Returns false if worker should exit.
+func (g *LogSizeGatherer) Work() bool {
+	var workItem WorkItem
+	select {
+	case <-g.stopChannel:
+		g.wg.Done()
+		return false
+	case workItem = <-g.workChannel:
+	}
+	sshResult, err := SSH(
+		fmt.Sprintf("ls -l %v | awk '{print $9, $5}' | tr '\n' ' '", strings.Join(workItem.paths, " ")),
+		workItem.ip,
+		TestContext.Provider,
+	)
+	if err != nil {
+		Logf("Error while trying to SSH to %v, skipping probe. Error: %v", workItem.ip, err)
+		// In case of repeated error give up.
+		if workItem.backoffMultiplier >= 128 {
+			Logf("Failed to ssh to a node %v multiple times in a row. Giving up.", workItem.ip)
+			g.wg.Done()
+			return false
+		}
+		workItem.backoffMultiplier *= 2
+		go g.pushWorkItem(workItem)
+		return true
+	}
+	workItem.backoffMultiplier = 1
+	results := strings.Split(sshResult.Stdout, " ")
+
+	now := time.Now()
+	for i := 0; i+1 < len(results); i = i + 2 {
+		path := results[i]
+		size, err := strconv.Atoi(results[i+1])
+		if err != nil {
+			Logf("Error during conversion to int: %v, skipping data. Error: %v", results[i+1], err)
+			continue
+		}
+		g.data.AddNewData(workItem.ip, path, now, size)
+	}
+	go g.pushWorkItem(workItem)
+	return true
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/api_server_metrics.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+// APIServerMetrics is metrics for API server
+type APIServerMetrics Metrics
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *APIServerMetrics) Equal(o APIServerMetrics) bool {
+	return (*Metrics)(m).Equal(Metrics(o))
+}
+
+func newAPIServerMetrics() APIServerMetrics {
+	result := NewMetrics()
+	return APIServerMetrics(result)
+}
+
+func parseAPIServerMetrics(data string) (APIServerMetrics, error) {
+	result := newAPIServerMetrics()
+	if err := parseMetrics(data, (*Metrics)(&result)); err != nil {
+		return APIServerMetrics{}, err
+	}
+	return result, nil
+}
+
+func (g *Grabber) getMetricsFromAPIServer() (string, error) {
+	rawOutput, err := g.client.CoreV1().RESTClient().Get().RequestURI("/metrics").Do().Raw()
+	if err != nil {
+		return "", err
+	}
+	return string(rawOutput), nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/cluster_autoscaler_metrics.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+// ClusterAutoscalerMetrics is metrics for cluster autoscaller
+type ClusterAutoscalerMetrics Metrics
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *ClusterAutoscalerMetrics) Equal(o ClusterAutoscalerMetrics) bool {
+	return (*Metrics)(m).Equal(Metrics(o))
+}
+
+func newClusterAutoscalerMetrics() ClusterAutoscalerMetrics {
+	result := NewMetrics()
+	return ClusterAutoscalerMetrics(result)
+}
+
+func parseClusterAutoscalerMetrics(data string) (ClusterAutoscalerMetrics, error) {
+	result := newClusterAutoscalerMetrics()
+	if err := parseMetrics(data, (*Metrics)(&result)); err != nil {
+		return ClusterAutoscalerMetrics{}, err
+	}
+	return result, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/controller_manager_metrics.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+// ControllerManagerMetrics is metrics for controller manager
+type ControllerManagerMetrics Metrics
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *ControllerManagerMetrics) Equal(o ControllerManagerMetrics) bool {
+	return (*Metrics)(m).Equal(Metrics(o))
+}
+
+func newControllerManagerMetrics() ControllerManagerMetrics {
+	result := NewMetrics()
+	return ControllerManagerMetrics(result)
+}
+
+func parseControllerManagerMetrics(data string) (ControllerManagerMetrics, error) {
+	result := newControllerManagerMetrics()
+	if err := parseMetrics(data, (*Metrics)(&result)); err != nil {
+		return ControllerManagerMetrics{}, err
+	}
+	return result, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/generic_metrics.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"io"
+	"reflect"
+	"strings"
+
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+	"k8s.io/klog"
+)
+
+// Metrics is generic metrics for other specific metrics
+type Metrics map[string]model.Samples
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *Metrics) Equal(o Metrics) bool {
+	leftKeySet := []string{}
+	rightKeySet := []string{}
+	for k := range *m {
+		leftKeySet = append(leftKeySet, k)
+	}
+	for k := range o {
+		rightKeySet = append(rightKeySet, k)
+	}
+	if !reflect.DeepEqual(leftKeySet, rightKeySet) {
+		return false
+	}
+	for _, k := range leftKeySet {
+		if !(*m)[k].Equal(o[k]) {
+			return false
+		}
+	}
+	return true
+}
+
+// NewMetrics returns new metrics which are initialized.
+func NewMetrics() Metrics {
+	result := make(Metrics)
+	return result
+}
+
+func parseMetrics(data string, output *Metrics) error {
+	dec := expfmt.NewDecoder(strings.NewReader(data), expfmt.FmtText)
+	decoder := expfmt.SampleDecoder{
+		Dec:  dec,
+		Opts: &expfmt.DecodeOptions{},
+	}
+
+	for {
+		var v model.Vector
+		if err := decoder.Decode(&v); err != nil {
+			if err == io.EOF {
+				// Expected loop termination condition.
+				return nil
+			}
+			klog.Warningf("Invalid Decode. Skipping.")
+			continue
+		}
+		for _, metric := range v {
+			name := string(metric.Metric[model.MetricNameLabel])
+			(*output)[name] = append((*output)[name], metric)
+		}
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/kubelet_metrics.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+const (
+	proxyTimeout = 2 * time.Minute
+)
+
+// KubeletMetrics is metrics for kubelet
+type KubeletMetrics Metrics
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *KubeletMetrics) Equal(o KubeletMetrics) bool {
+	return (*Metrics)(m).Equal(Metrics(o))
+}
+
+// NewKubeletMetrics returns new metrics which are initialized.
+func NewKubeletMetrics() KubeletMetrics {
+	result := NewMetrics()
+	return KubeletMetrics(result)
+}
+
+// GrabKubeletMetricsWithoutProxy retrieve metrics from the kubelet on the given node using a simple GET over http.
+// Currently only used in integration tests.
+func GrabKubeletMetricsWithoutProxy(nodeName, path string) (KubeletMetrics, error) {
+	resp, err := http.Get(fmt.Sprintf("http://%s%s", nodeName, path))
+	if err != nil {
+		return KubeletMetrics{}, err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return KubeletMetrics{}, err
+	}
+	return parseKubeletMetrics(string(body))
+}
+
+func parseKubeletMetrics(data string) (KubeletMetrics, error) {
+	result := NewKubeletMetrics()
+	if err := parseMetrics(data, (*Metrics)(&result)); err != nil {
+		return KubeletMetrics{}, err
+	}
+	return result, nil
+}
+
+func (g *Grabber) getMetricsFromNode(nodeName string, kubeletPort int) (string, error) {
+	// There's a problem with timing out during proxy. Wrapping this in a goroutine to prevent deadlock.
+	// Hanging goroutine will be leaked.
+	finished := make(chan struct{})
+	var err error
+	var rawOutput []byte
+	go func() {
+		rawOutput, err = g.client.CoreV1().RESTClient().Get().
+			Resource("nodes").
+			SubResource("proxy").
+			Name(fmt.Sprintf("%v:%v", nodeName, kubeletPort)).
+			Suffix("metrics").
+			Do().Raw()
+		finished <- struct{}{}
+	}()
+	select {
+	case <-time.After(proxyTimeout):
+		return "", fmt.Errorf("Timed out when waiting for proxy to gather metrics from %v", nodeName)
+	case <-finished:
+		if err != nil {
+			return "", err
+		}
+		return string(rawOutput), nil
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/metrics_grabber.go

@@ -0,0 +1,249 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	clientset "k8s.io/client-go/kubernetes"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/pkg/util/system"
+
+	"k8s.io/klog"
+)
+
+// Collection is metrics collection of components
+type Collection struct {
+	APIServerMetrics         APIServerMetrics
+	ControllerManagerMetrics ControllerManagerMetrics
+	KubeletMetrics           map[string]KubeletMetrics
+	SchedulerMetrics         SchedulerMetrics
+	ClusterAutoscalerMetrics ClusterAutoscalerMetrics
+}
+
+// Grabber provides functions which grab metrics from components
+type Grabber struct {
+	client                    clientset.Interface
+	externalClient            clientset.Interface
+	grabFromAPIServer         bool
+	grabFromControllerManager bool
+	grabFromKubelets          bool
+	grabFromScheduler         bool
+	grabFromClusterAutoscaler bool
+	masterName                string
+	registeredMaster          bool
+}
+
+// NewMetricsGrabber returns new metrics which are initialized.
+func NewMetricsGrabber(c clientset.Interface, ec clientset.Interface, kubelets bool, scheduler bool, controllers bool, apiServer bool, clusterAutoscaler bool) (*Grabber, error) {
+	registeredMaster := false
+	masterName := ""
+	nodeList, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	if len(nodeList.Items) < 1 {
+		klog.Warning("Can't find any Nodes in the API server to grab metrics from")
+	}
+	for _, node := range nodeList.Items {
+		if system.IsMasterNode(node.Name) {
+			registeredMaster = true
+			masterName = node.Name
+			break
+		}
+	}
+	if !registeredMaster {
+		scheduler = false
+		controllers = false
+		clusterAutoscaler = ec != nil
+		if clusterAutoscaler {
+			klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager is disabled.")
+		} else {
+			klog.Warningf("Master node is not registered. Grabbing metrics from Scheduler, ControllerManager and ClusterAutoscaler is disabled.")
+		}
+	}
+
+	return &Grabber{
+		client:                    c,
+		externalClient:            ec,
+		grabFromAPIServer:         apiServer,
+		grabFromControllerManager: controllers,
+		grabFromKubelets:          kubelets,
+		grabFromScheduler:         scheduler,
+		grabFromClusterAutoscaler: clusterAutoscaler,
+		masterName:                masterName,
+		registeredMaster:          registeredMaster,
+	}, nil
+}
+
+// HasRegisteredMaster returns if metrics grabber was able to find a master node
+func (g *Grabber) HasRegisteredMaster() bool {
+	return g.registeredMaster
+}
+
+// GrabFromKubelet returns metrics from kubelet
+func (g *Grabber) GrabFromKubelet(nodeName string) (KubeletMetrics, error) {
+	nodes, err := g.client.CoreV1().Nodes().List(metav1.ListOptions{FieldSelector: fields.Set{api.ObjectNameField: nodeName}.AsSelector().String()})
+	if err != nil {
+		return KubeletMetrics{}, err
+	}
+	if len(nodes.Items) != 1 {
+		return KubeletMetrics{}, fmt.Errorf("Error listing nodes with name %v, got %v", nodeName, nodes.Items)
+	}
+	kubeletPort := nodes.Items[0].Status.DaemonEndpoints.KubeletEndpoint.Port
+	return g.grabFromKubeletInternal(nodeName, int(kubeletPort))
+}
+
+func (g *Grabber) grabFromKubeletInternal(nodeName string, kubeletPort int) (KubeletMetrics, error) {
+	if kubeletPort <= 0 || kubeletPort > 65535 {
+		return KubeletMetrics{}, fmt.Errorf("Invalid Kubelet port %v. Skipping Kubelet's metrics gathering", kubeletPort)
+	}
+	output, err := g.getMetricsFromNode(nodeName, int(kubeletPort))
+	if err != nil {
+		return KubeletMetrics{}, err
+	}
+	return parseKubeletMetrics(output)
+}
+
+// GrabFromScheduler returns metrics from scheduler
+func (g *Grabber) GrabFromScheduler() (SchedulerMetrics, error) {
+	if !g.registeredMaster {
+		return SchedulerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping Scheduler's metrics gathering")
+	}
+	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-scheduler", g.masterName), metav1.NamespaceSystem, ports.InsecureSchedulerPort)
+	if err != nil {
+		return SchedulerMetrics{}, err
+	}
+	return parseSchedulerMetrics(output)
+}
+
+// GrabFromClusterAutoscaler returns metrics from cluster autoscaler
+func (g *Grabber) GrabFromClusterAutoscaler() (ClusterAutoscalerMetrics, error) {
+	if !g.registeredMaster && g.externalClient == nil {
+		return ClusterAutoscalerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ClusterAutoscaler's metrics gathering")
+	}
+	var client clientset.Interface
+	var namespace string
+	if g.externalClient != nil {
+		client = g.externalClient
+		namespace = "kubemark"
+	} else {
+		client = g.client
+		namespace = metav1.NamespaceSystem
+	}
+	output, err := g.getMetricsFromPod(client, "cluster-autoscaler", namespace, 8085)
+	if err != nil {
+		return ClusterAutoscalerMetrics{}, err
+	}
+	return parseClusterAutoscalerMetrics(output)
+}
+
+// GrabFromControllerManager returns metrics from controller manager
+func (g *Grabber) GrabFromControllerManager() (ControllerManagerMetrics, error) {
+	if !g.registeredMaster {
+		return ControllerManagerMetrics{}, fmt.Errorf("Master's Kubelet is not registered. Skipping ControllerManager's metrics gathering")
+	}
+	output, err := g.getMetricsFromPod(g.client, fmt.Sprintf("%v-%v", "kube-controller-manager", g.masterName), metav1.NamespaceSystem, ports.InsecureKubeControllerManagerPort)
+	if err != nil {
+		return ControllerManagerMetrics{}, err
+	}
+	return parseControllerManagerMetrics(output)
+}
+
+// GrabFromAPIServer returns metrics from API server
+func (g *Grabber) GrabFromAPIServer() (APIServerMetrics, error) {
+	output, err := g.getMetricsFromAPIServer()
+	if err != nil {
+		return APIServerMetrics{}, nil
+	}
+	return parseAPIServerMetrics(output)
+}
+
+// Grab returns metrics from corresponding component
+func (g *Grabber) Grab() (Collection, error) {
+	result := Collection{}
+	var errs []error
+	if g.grabFromAPIServer {
+		metrics, err := g.GrabFromAPIServer()
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			result.APIServerMetrics = metrics
+		}
+	}
+	if g.grabFromScheduler {
+		metrics, err := g.GrabFromScheduler()
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			result.SchedulerMetrics = metrics
+		}
+	}
+	if g.grabFromControllerManager {
+		metrics, err := g.GrabFromControllerManager()
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			result.ControllerManagerMetrics = metrics
+		}
+	}
+	if g.grabFromClusterAutoscaler {
+		metrics, err := g.GrabFromClusterAutoscaler()
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			result.ClusterAutoscalerMetrics = metrics
+		}
+	}
+	if g.grabFromKubelets {
+		result.KubeletMetrics = make(map[string]KubeletMetrics)
+		nodes, err := g.client.CoreV1().Nodes().List(metav1.ListOptions{})
+		if err != nil {
+			errs = append(errs, err)
+		} else {
+			for _, node := range nodes.Items {
+				kubeletPort := node.Status.DaemonEndpoints.KubeletEndpoint.Port
+				metrics, err := g.grabFromKubeletInternal(node.Name, int(kubeletPort))
+				if err != nil {
+					errs = append(errs, err)
+				}
+				result.KubeletMetrics[node.Name] = metrics
+			}
+		}
+	}
+	if len(errs) > 0 {
+		return result, fmt.Errorf("Errors while grabbing metrics: %v", errs)
+	}
+	return result, nil
+}
+
+func (g *Grabber) getMetricsFromPod(client clientset.Interface, podName string, namespace string, port int) (string, error) {
+	rawOutput, err := client.CoreV1().RESTClient().Get().
+		Namespace(namespace).
+		Resource("pods").
+		SubResource("proxy").
+		Name(fmt.Sprintf("%v:%v", podName, port)).
+		Suffix("metrics").
+		Do().Raw()
+	if err != nil {
+		return "", err
+	}
+	return string(rawOutput), nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics/scheduler_metrics.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+// SchedulerMetrics is metrics for scheduler
+type SchedulerMetrics Metrics
+
+// Equal returns true if all metrics are the same as the arguments.
+func (m *SchedulerMetrics) Equal(o SchedulerMetrics) bool {
+	return (*Metrics)(m).Equal(Metrics(o))
+}
+
+func newSchedulerMetrics() SchedulerMetrics {
+	result := NewMetrics()
+	return SchedulerMetrics(result)
+}
+
+func parseSchedulerMetrics(data string) (SchedulerMetrics, error) {
+	result := newSchedulerMetrics()
+	if err := parseMetrics(data, (*Metrics)(&result)); err != nil {
+		return SchedulerMetrics{}, err
+	}
+	return result, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics_util.go

@@ -0,0 +1,836 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/master/ports"
+	schedulermetric "k8s.io/kubernetes/pkg/scheduler/metrics"
+	"k8s.io/kubernetes/pkg/util/system"
+	"k8s.io/kubernetes/test/e2e/framework/metrics"
+
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+)
+
+const (
+	// NodeStartupThreshold is a rough estimate of the time allocated for a pod to start on a node.
+	NodeStartupThreshold = 4 * time.Second
+
+	// We are setting 1s threshold for apicalls even in small clusters to avoid flakes.
+	// The problem is that if long GC is happening in small clusters (where we have e.g.
+	// 1-core master machines) and tests are pretty short, it may consume significant
+	// portion of CPU and basically stop all the real work.
+	// Increasing threshold to 1s is within our SLO and should solve this problem.
+	apiCallLatencyThreshold time.Duration = 1 * time.Second
+
+	// We use a higher threshold for list apicalls if the cluster is big (i.e having > 500 nodes)
+	// as list response sizes are bigger in general for big clusters. We also use a higher threshold
+	// for list calls at cluster scope (this includes non-namespaced and all-namespaced calls).
+	apiListCallLatencyThreshold      time.Duration = 5 * time.Second
+	apiClusterScopeListCallThreshold time.Duration = 10 * time.Second
+	bigClusterNodeCountThreshold                   = 500
+
+	// Cluster Autoscaler metrics names
+	caFunctionMetric      = "cluster_autoscaler_function_duration_seconds_bucket"
+	caFunctionMetricLabel = "function"
+)
+
+type MetricsForE2E metrics.Collection
+
+func (m *MetricsForE2E) filterMetrics() {
+	interestingAPIServerMetrics := make(metrics.APIServerMetrics)
+	for _, metric := range InterestingAPIServerMetrics {
+		interestingAPIServerMetrics[metric] = (*m).APIServerMetrics[metric]
+	}
+	interestingControllerManagerMetrics := make(metrics.ControllerManagerMetrics)
+	for _, metric := range InterestingControllerManagerMetrics {
+		interestingControllerManagerMetrics[metric] = (*m).ControllerManagerMetrics[metric]
+	}
+	interestingClusterAutoscalerMetrics := make(metrics.ClusterAutoscalerMetrics)
+	for _, metric := range InterestingClusterAutoscalerMetrics {
+		interestingClusterAutoscalerMetrics[metric] = (*m).ClusterAutoscalerMetrics[metric]
+	}
+	interestingKubeletMetrics := make(map[string]metrics.KubeletMetrics)
+	for kubelet, grabbed := range (*m).KubeletMetrics {
+		interestingKubeletMetrics[kubelet] = make(metrics.KubeletMetrics)
+		for _, metric := range InterestingKubeletMetrics {
+			interestingKubeletMetrics[kubelet][metric] = grabbed[metric]
+		}
+	}
+	(*m).APIServerMetrics = interestingAPIServerMetrics
+	(*m).ControllerManagerMetrics = interestingControllerManagerMetrics
+	(*m).KubeletMetrics = interestingKubeletMetrics
+}
+
+func printSample(sample *model.Sample) string {
+	buf := make([]string, 0)
+	// Id is a VERY special label. For 'normal' container it's useless, but it's necessary
+	// for 'system' containers (e.g. /docker-daemon, /kubelet, etc.). We know if that's the
+	// case by checking if there's a label "kubernetes_container_name" present. It's hacky
+	// but it works...
+	_, normalContainer := sample.Metric["kubernetes_container_name"]
+	for k, v := range sample.Metric {
+		if strings.HasPrefix(string(k), "__") {
+			continue
+		}
+
+		if string(k) == "id" && normalContainer {
+			continue
+		}
+		buf = append(buf, fmt.Sprintf("%v=%v", string(k), v))
+	}
+	return fmt.Sprintf("[%v] = %v", strings.Join(buf, ","), sample.Value)
+}
+
+func (m *MetricsForE2E) PrintHumanReadable() string {
+	buf := bytes.Buffer{}
+	for _, interestingMetric := range InterestingAPIServerMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", interestingMetric))
+		for _, sample := range (*m).APIServerMetrics[interestingMetric] {
+			buf.WriteString(fmt.Sprintf("\t%v\n", printSample(sample)))
+		}
+	}
+	for _, interestingMetric := range InterestingControllerManagerMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", interestingMetric))
+		for _, sample := range (*m).ControllerManagerMetrics[interestingMetric] {
+			buf.WriteString(fmt.Sprintf("\t%v\n", printSample(sample)))
+		}
+	}
+	for _, interestingMetric := range InterestingClusterAutoscalerMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", interestingMetric))
+		for _, sample := range (*m).ClusterAutoscalerMetrics[interestingMetric] {
+			buf.WriteString(fmt.Sprintf("\t%v\n", printSample(sample)))
+		}
+	}
+	for kubelet, grabbed := range (*m).KubeletMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", kubelet))
+		for _, interestingMetric := range InterestingKubeletMetrics {
+			buf.WriteString(fmt.Sprintf("\tFor %v:\n", interestingMetric))
+			for _, sample := range grabbed[interestingMetric] {
+				buf.WriteString(fmt.Sprintf("\t\t%v\n", printSample(sample)))
+			}
+		}
+	}
+	return buf.String()
+}
+
+func (m *MetricsForE2E) PrintJSON() string {
+	m.filterMetrics()
+	return PrettyPrintJSON(m)
+}
+
+func (m *MetricsForE2E) SummaryKind() string {
+	return "MetricsForE2E"
+}
+
+var SchedulingLatencyMetricName = model.LabelValue(schedulermetric.SchedulerSubsystem + "_" + schedulermetric.SchedulingLatencyName)
+
+var InterestingAPIServerMetrics = []string{
+	"apiserver_request_total",
+	// TODO(krzysied): apiserver_request_latencies_summary is a deprecated metric.
+	// It should be replaced with new metric.
+	"apiserver_request_latencies_summary",
+	"apiserver_init_events_total",
+}
+
+var InterestingControllerManagerMetrics = []string{
+	"garbage_collector_attempt_to_delete_queue_latency",
+	"garbage_collector_attempt_to_delete_work_duration",
+	"garbage_collector_attempt_to_orphan_queue_latency",
+	"garbage_collector_attempt_to_orphan_work_duration",
+	"garbage_collector_dirty_processing_latency_microseconds",
+	"garbage_collector_event_processing_latency_microseconds",
+	"garbage_collector_graph_changes_queue_latency",
+	"garbage_collector_graph_changes_work_duration",
+	"garbage_collector_orphan_processing_latency_microseconds",
+
+	"namespace_queue_latency",
+	"namespace_queue_latency_sum",
+	"namespace_queue_latency_count",
+	"namespace_retries",
+	"namespace_work_duration",
+	"namespace_work_duration_sum",
+	"namespace_work_duration_count",
+}
+
+var InterestingKubeletMetrics = []string{
+	"kubelet_container_manager_latency_microseconds",
+	"kubelet_docker_errors",
+	"kubelet_docker_operations_duration_seconds",
+	"kubelet_generate_pod_status_latency_microseconds",
+	"kubelet_pod_start_duration_seconds",
+	"kubelet_pod_worker_duration_seconds",
+	"kubelet_pod_worker_start_duration_seconds",
+	"kubelet_sync_pods_latency_microseconds",
+}
+
+var InterestingClusterAutoscalerMetrics = []string{
+	"function_duration_seconds",
+	"errors_total",
+	"evicted_pods_total",
+}
+
+// Dashboard metrics
+type LatencyMetric struct {
+	Perc50  time.Duration `json:"Perc50"`
+	Perc90  time.Duration `json:"Perc90"`
+	Perc99  time.Duration `json:"Perc99"`
+	Perc100 time.Duration `json:"Perc100"`
+}
+
+type PodStartupLatency struct {
+	CreateToScheduleLatency LatencyMetric `json:"createToScheduleLatency"`
+	ScheduleToRunLatency    LatencyMetric `json:"scheduleToRunLatency"`
+	RunToWatchLatency       LatencyMetric `json:"runToWatchLatency"`
+	ScheduleToWatchLatency  LatencyMetric `json:"scheduleToWatchLatency"`
+	E2ELatency              LatencyMetric `json:"e2eLatency"`
+}
+
+func (l *PodStartupLatency) SummaryKind() string {
+	return "PodStartupLatency"
+}
+
+func (l *PodStartupLatency) PrintHumanReadable() string {
+	return PrettyPrintJSON(l)
+}
+
+func (l *PodStartupLatency) PrintJSON() string {
+	return PrettyPrintJSON(PodStartupLatencyToPerfData(l))
+}
+
+type SchedulingMetrics struct {
+	PredicateEvaluationLatency  LatencyMetric `json:"predicateEvaluationLatency"`
+	PriorityEvaluationLatency   LatencyMetric `json:"priorityEvaluationLatency"`
+	PreemptionEvaluationLatency LatencyMetric `json:"preemptionEvaluationLatency"`
+	BindingLatency              LatencyMetric `json:"bindingLatency"`
+	ThroughputAverage           float64       `json:"throughputAverage"`
+	ThroughputPerc50            float64       `json:"throughputPerc50"`
+	ThroughputPerc90            float64       `json:"throughputPerc90"`
+	ThroughputPerc99            float64       `json:"throughputPerc99"`
+}
+
+func (l *SchedulingMetrics) SummaryKind() string {
+	return "SchedulingMetrics"
+}
+
+func (l *SchedulingMetrics) PrintHumanReadable() string {
+	return PrettyPrintJSON(l)
+}
+
+func (l *SchedulingMetrics) PrintJSON() string {
+	return PrettyPrintJSON(l)
+}
+
+type Histogram struct {
+	Labels  map[string]string `json:"labels"`
+	Buckets map[string]int    `json:"buckets"`
+}
+
+type HistogramVec []Histogram
+
+func newHistogram(labels map[string]string) *Histogram {
+	return &Histogram{
+		Labels:  labels,
+		Buckets: make(map[string]int),
+	}
+}
+
+type EtcdMetrics struct {
+	BackendCommitDuration     HistogramVec `json:"backendCommitDuration"`
+	SnapshotSaveTotalDuration HistogramVec `json:"snapshotSaveTotalDuration"`
+	PeerRoundTripTime         HistogramVec `json:"peerRoundTripTime"`
+	WalFsyncDuration          HistogramVec `json:"walFsyncDuration"`
+	MaxDatabaseSize           float64      `json:"maxDatabaseSize"`
+}
+
+func newEtcdMetrics() *EtcdMetrics {
+	return &EtcdMetrics{
+		BackendCommitDuration:     make(HistogramVec, 0),
+		SnapshotSaveTotalDuration: make(HistogramVec, 0),
+		PeerRoundTripTime:         make(HistogramVec, 0),
+		WalFsyncDuration:          make(HistogramVec, 0),
+	}
+}
+
+func (l *EtcdMetrics) SummaryKind() string {
+	return "EtcdMetrics"
+}
+
+func (l *EtcdMetrics) PrintHumanReadable() string {
+	return PrettyPrintJSON(l)
+}
+
+func (l *EtcdMetrics) PrintJSON() string {
+	return PrettyPrintJSON(l)
+}
+
+type EtcdMetricsCollector struct {
+	stopCh  chan struct{}
+	wg      *sync.WaitGroup
+	metrics *EtcdMetrics
+}
+
+func NewEtcdMetricsCollector() *EtcdMetricsCollector {
+	return &EtcdMetricsCollector{
+		stopCh:  make(chan struct{}),
+		wg:      &sync.WaitGroup{},
+		metrics: newEtcdMetrics(),
+	}
+}
+
+func getEtcdMetrics() ([]*model.Sample, error) {
+	// Etcd is only exposed on localhost level. We are using ssh method
+	if TestContext.Provider == "gke" || TestContext.Provider == "eks" {
+		Logf("Not grabbing etcd metrics through master SSH: unsupported for %s", TestContext.Provider)
+		return nil, nil
+	}
+
+	cmd := "curl http://localhost:2379/metrics"
+	sshResult, err := SSH(cmd, GetMasterHost()+":22", TestContext.Provider)
+	if err != nil || sshResult.Code != 0 {
+		return nil, fmt.Errorf("unexpected error (code: %d) in ssh connection to master: %#v", sshResult.Code, err)
+	}
+	data := sshResult.Stdout
+
+	return extractMetricSamples(data)
+}
+
+func getEtcdDatabaseSize() (float64, error) {
+	samples, err := getEtcdMetrics()
+	if err != nil {
+		return 0, err
+	}
+	for _, sample := range samples {
+		if sample.Metric[model.MetricNameLabel] == "etcd_debugging_mvcc_db_total_size_in_bytes" {
+			return float64(sample.Value), nil
+		}
+	}
+	return 0, fmt.Errorf("Couldn't find etcd database size metric")
+}
+
+// StartCollecting starts to collect etcd db size metric periodically
+// and updates MaxDatabaseSize accordingly.
+func (mc *EtcdMetricsCollector) StartCollecting(interval time.Duration) {
+	mc.wg.Add(1)
+	go func() {
+		defer mc.wg.Done()
+		for {
+			select {
+			case <-time.After(interval):
+				dbSize, err := getEtcdDatabaseSize()
+				if err != nil {
+					Logf("Failed to collect etcd database size")
+					continue
+				}
+				mc.metrics.MaxDatabaseSize = math.Max(mc.metrics.MaxDatabaseSize, dbSize)
+			case <-mc.stopCh:
+				return
+			}
+		}
+	}()
+}
+
+func (mc *EtcdMetricsCollector) StopAndSummarize() error {
+	close(mc.stopCh)
+	mc.wg.Wait()
+
+	// Do some one-off collection of metrics.
+	samples, err := getEtcdMetrics()
+	if err != nil {
+		return err
+	}
+	for _, sample := range samples {
+		switch sample.Metric[model.MetricNameLabel] {
+		case "etcd_disk_backend_commit_duration_seconds_bucket":
+			convertSampleToBucket(sample, &mc.metrics.BackendCommitDuration)
+		case "etcd_debugging_snap_save_total_duration_seconds_bucket":
+			convertSampleToBucket(sample, &mc.metrics.SnapshotSaveTotalDuration)
+		case "etcd_disk_wal_fsync_duration_seconds_bucket":
+			convertSampleToBucket(sample, &mc.metrics.WalFsyncDuration)
+		case "etcd_network_peer_round_trip_time_seconds_bucket":
+			convertSampleToBucket(sample, &mc.metrics.PeerRoundTripTime)
+		}
+	}
+	return nil
+}
+
+func (mc *EtcdMetricsCollector) GetMetrics() *EtcdMetrics {
+	return mc.metrics
+}
+
+type SaturationTime struct {
+	TimeToSaturate time.Duration `json:"timeToSaturate"`
+	NumberOfNodes  int           `json:"numberOfNodes"`
+	NumberOfPods   int           `json:"numberOfPods"`
+	Throughput     float32       `json:"throughput"`
+}
+
+type APICall struct {
+	Resource    string        `json:"resource"`
+	Subresource string        `json:"subresource"`
+	Verb        string        `json:"verb"`
+	Scope       string        `json:"scope"`
+	Latency     LatencyMetric `json:"latency"`
+	Count       int           `json:"count"`
+}
+
+type APIResponsiveness struct {
+	APICalls []APICall `json:"apicalls"`
+}
+
+func (a *APIResponsiveness) SummaryKind() string {
+	return "APIResponsiveness"
+}
+
+func (a *APIResponsiveness) PrintHumanReadable() string {
+	return PrettyPrintJSON(a)
+}
+
+func (a *APIResponsiveness) PrintJSON() string {
+	return PrettyPrintJSON(ApiCallToPerfData(a))
+}
+
+func (a *APIResponsiveness) Len() int { return len(a.APICalls) }
+func (a *APIResponsiveness) Swap(i, j int) {
+	a.APICalls[i], a.APICalls[j] = a.APICalls[j], a.APICalls[i]
+}
+func (a *APIResponsiveness) Less(i, j int) bool {
+	return a.APICalls[i].Latency.Perc99 < a.APICalls[j].Latency.Perc99
+}
+
+// Set request latency for a particular quantile in the APICall metric entry (creating one if necessary).
+// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func (a *APIResponsiveness) addMetricRequestLatency(resource, subresource, verb, scope string, quantile float64, latency time.Duration) {
+	for i, apicall := range a.APICalls {
+		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
+			a.APICalls[i] = setQuantileAPICall(apicall, quantile, latency)
+			return
+		}
+	}
+	apicall := setQuantileAPICall(APICall{Resource: resource, Subresource: subresource, Verb: verb, Scope: scope}, quantile, latency)
+	a.APICalls = append(a.APICalls, apicall)
+}
+
+// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func setQuantileAPICall(apicall APICall, quantile float64, latency time.Duration) APICall {
+	setQuantile(&apicall.Latency, quantile, latency)
+	return apicall
+}
+
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func setQuantile(metric *LatencyMetric, quantile float64, latency time.Duration) {
+	switch quantile {
+	case 0.5:
+		metric.Perc50 = latency
+	case 0.9:
+		metric.Perc90 = latency
+	case 0.99:
+		metric.Perc99 = latency
+	}
+}
+
+// Add request count to the APICall metric entry (creating one if necessary).
+func (a *APIResponsiveness) addMetricRequestCount(resource, subresource, verb, scope string, count int) {
+	for i, apicall := range a.APICalls {
+		if apicall.Resource == resource && apicall.Subresource == subresource && apicall.Verb == verb && apicall.Scope == scope {
+			a.APICalls[i].Count += count
+			return
+		}
+	}
+	apicall := APICall{Resource: resource, Subresource: subresource, Verb: verb, Count: count, Scope: scope}
+	a.APICalls = append(a.APICalls, apicall)
+}
+
+func readLatencyMetrics(c clientset.Interface) (*APIResponsiveness, error) {
+	var a APIResponsiveness
+
+	body, err := getMetrics(c)
+	if err != nil {
+		return nil, err
+	}
+
+	samples, err := extractMetricSamples(body)
+	if err != nil {
+		return nil, err
+	}
+
+	ignoredResources := sets.NewString("events")
+	// TODO: figure out why we're getting non-capitalized proxy and fix this.
+	ignoredVerbs := sets.NewString("WATCH", "WATCHLIST", "PROXY", "proxy", "CONNECT")
+
+	for _, sample := range samples {
+		// Example line:
+		// apiserver_request_latencies_summary{resource="namespaces",verb="LIST",quantile="0.99"} 908
+		// apiserver_request_total{resource="pods",verb="LIST",client="kubectl",code="200",contentType="json"} 233
+		if sample.Metric[model.MetricNameLabel] != "apiserver_request_latencies_summary" &&
+			sample.Metric[model.MetricNameLabel] != "apiserver_request_total" {
+			continue
+		}
+
+		resource := string(sample.Metric["resource"])
+		subresource := string(sample.Metric["subresource"])
+		verb := string(sample.Metric["verb"])
+		scope := string(sample.Metric["scope"])
+		if ignoredResources.Has(resource) || ignoredVerbs.Has(verb) {
+			continue
+		}
+
+		switch sample.Metric[model.MetricNameLabel] {
+		case "apiserver_request_latencies_summary":
+			latency := sample.Value
+			quantile, err := strconv.ParseFloat(string(sample.Metric[model.QuantileLabel]), 64)
+			if err != nil {
+				return nil, err
+			}
+			a.addMetricRequestLatency(resource, subresource, verb, scope, quantile, time.Duration(int64(latency))*time.Microsecond)
+		case "apiserver_request_total":
+			count := sample.Value
+			a.addMetricRequestCount(resource, subresource, verb, scope, int(count))
+
+		}
+	}
+
+	return &a, err
+}
+
+// Prints top five summary metrics for request types with latency and returns
+// number of such request types above threshold. We use a higher threshold for
+// list calls if nodeCount is above a given threshold (i.e. cluster is big).
+func HighLatencyRequests(c clientset.Interface, nodeCount int) (int, *APIResponsiveness, error) {
+	isBigCluster := (nodeCount > bigClusterNodeCountThreshold)
+	metrics, err := readLatencyMetrics(c)
+	if err != nil {
+		return 0, metrics, err
+	}
+	sort.Sort(sort.Reverse(metrics))
+	badMetrics := 0
+	top := 5
+	for i := range metrics.APICalls {
+		latency := metrics.APICalls[i].Latency.Perc99
+		isListCall := (metrics.APICalls[i].Verb == "LIST")
+		isClusterScopedCall := (metrics.APICalls[i].Scope == "cluster")
+		isBad := false
+		latencyThreshold := apiCallLatencyThreshold
+		if isListCall && isBigCluster {
+			latencyThreshold = apiListCallLatencyThreshold
+			if isClusterScopedCall {
+				latencyThreshold = apiClusterScopeListCallThreshold
+			}
+		}
+		if latency > latencyThreshold {
+			isBad = true
+			badMetrics++
+		}
+		if top > 0 || isBad {
+			top--
+			prefix := ""
+			if isBad {
+				prefix = "WARNING "
+			}
+			Logf("%vTop latency metric: %+v", prefix, metrics.APICalls[i])
+		}
+	}
+	return badMetrics, metrics, nil
+}
+
+// Verifies whether 50, 90 and 99th percentiles of a latency metric are
+// within the expected threshold.
+func VerifyLatencyWithinThreshold(threshold, actual LatencyMetric, metricName string) error {
+	if actual.Perc50 > threshold.Perc50 {
+		return fmt.Errorf("too high %v latency 50th percentile: %v", metricName, actual.Perc50)
+	}
+	if actual.Perc90 > threshold.Perc90 {
+		return fmt.Errorf("too high %v latency 90th percentile: %v", metricName, actual.Perc90)
+	}
+	if actual.Perc99 > threshold.Perc99 {
+		return fmt.Errorf("too high %v latency 99th percentile: %v", metricName, actual.Perc99)
+	}
+	return nil
+}
+
+// Resets latency metrics in apiserver.
+func ResetMetrics(c clientset.Interface) error {
+	Logf("Resetting latency metrics in apiserver...")
+	body, err := c.CoreV1().RESTClient().Delete().AbsPath("/metrics").DoRaw()
+	if err != nil {
+		return err
+	}
+	if string(body) != "metrics reset\n" {
+		return fmt.Errorf("Unexpected response: %q", string(body))
+	}
+	return nil
+}
+
+// Retrieves metrics information.
+func getMetrics(c clientset.Interface) (string, error) {
+	body, err := c.CoreV1().RESTClient().Get().AbsPath("/metrics").DoRaw()
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+// Sends REST request to kube scheduler metrics
+func sendRestRequestToScheduler(c clientset.Interface, op string) (string, error) {
+	opUpper := strings.ToUpper(op)
+	if opUpper != "GET" && opUpper != "DELETE" {
+		return "", fmt.Errorf("Unknown REST request")
+	}
+
+	nodes, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
+	ExpectNoError(err)
+
+	var masterRegistered = false
+	for _, node := range nodes.Items {
+		if system.IsMasterNode(node.Name) {
+			masterRegistered = true
+		}
+	}
+
+	var responseText string
+	if masterRegistered {
+		ctx, cancel := context.WithTimeout(context.Background(), SingleCallTimeout)
+		defer cancel()
+
+		body, err := c.CoreV1().RESTClient().Verb(opUpper).
+			Context(ctx).
+			Namespace(metav1.NamespaceSystem).
+			Resource("pods").
+			Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.InsecureSchedulerPort)).
+			SubResource("proxy").
+			Suffix("metrics").
+			Do().Raw()
+
+		ExpectNoError(err)
+		responseText = string(body)
+	} else {
+		// If master is not registered fall back to old method of using SSH.
+		if TestContext.Provider == "gke" || TestContext.Provider == "eks" {
+			Logf("Not grabbing scheduler metrics through master SSH: unsupported for %s", TestContext.Provider)
+			return "", nil
+		}
+
+		cmd := "curl -X " + opUpper + " http://localhost:10251/metrics"
+		sshResult, err := SSH(cmd, GetMasterHost()+":22", TestContext.Provider)
+		if err != nil || sshResult.Code != 0 {
+			return "", fmt.Errorf("unexpected error (code: %d) in ssh connection to master: %#v", sshResult.Code, err)
+		}
+		responseText = sshResult.Stdout
+	}
+	return responseText, nil
+}
+
+// Retrieves scheduler latency metrics.
+func getSchedulingLatency(c clientset.Interface) (*SchedulingMetrics, error) {
+	result := SchedulingMetrics{}
+	data, err := sendRestRequestToScheduler(c, "GET")
+	if err != nil {
+		return nil, err
+	}
+
+	samples, err := extractMetricSamples(data)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, sample := range samples {
+		if sample.Metric[model.MetricNameLabel] != SchedulingLatencyMetricName {
+			continue
+		}
+
+		var metric *LatencyMetric = nil
+		switch sample.Metric[schedulermetric.OperationLabel] {
+		case schedulermetric.PredicateEvaluation:
+			metric = &result.PredicateEvaluationLatency
+		case schedulermetric.PriorityEvaluation:
+			metric = &result.PriorityEvaluationLatency
+		case schedulermetric.PreemptionEvaluation:
+			metric = &result.PreemptionEvaluationLatency
+		case schedulermetric.Binding:
+			metric = &result.BindingLatency
+		}
+		if metric == nil {
+			continue
+		}
+
+		quantile, err := strconv.ParseFloat(string(sample.Metric[model.QuantileLabel]), 64)
+		if err != nil {
+			return nil, err
+		}
+		setQuantile(metric, quantile, time.Duration(int64(float64(sample.Value)*float64(time.Second))))
+	}
+	return &result, nil
+}
+
+// Verifies (currently just by logging them) the scheduling latencies.
+func VerifySchedulerLatency(c clientset.Interface) (*SchedulingMetrics, error) {
+	latency, err := getSchedulingLatency(c)
+	if err != nil {
+		return nil, err
+	}
+	return latency, nil
+}
+
+func ResetSchedulerMetrics(c clientset.Interface) error {
+	responseText, err := sendRestRequestToScheduler(c, "DELETE")
+	if err != nil {
+		return fmt.Errorf("Unexpected response: %q", responseText)
+	}
+	return nil
+}
+
+func convertSampleToBucket(sample *model.Sample, h *HistogramVec) {
+	labels := make(map[string]string)
+	for k, v := range sample.Metric {
+		if k != "le" {
+			labels[string(k)] = string(v)
+		}
+	}
+	var hist *Histogram
+	for i := range *h {
+		if reflect.DeepEqual(labels, (*h)[i].Labels) {
+			hist = &((*h)[i])
+			break
+		}
+	}
+	if hist == nil {
+		hist = newHistogram(labels)
+		*h = append(*h, *hist)
+	}
+	hist.Buckets[string(sample.Metric["le"])] = int(sample.Value)
+}
+
+func PrettyPrintJSON(metrics interface{}) string {
+	output := &bytes.Buffer{}
+	if err := json.NewEncoder(output).Encode(metrics); err != nil {
+		Logf("Error building encoder: %v", err)
+		return ""
+	}
+	formatted := &bytes.Buffer{}
+	if err := json.Indent(formatted, output.Bytes(), "", "  "); err != nil {
+		Logf("Error indenting: %v", err)
+		return ""
+	}
+	return string(formatted.Bytes())
+}
+
+// extractMetricSamples parses the prometheus metric samples from the input string.
+func extractMetricSamples(metricsBlob string) ([]*model.Sample, error) {
+	dec := expfmt.NewDecoder(strings.NewReader(metricsBlob), expfmt.FmtText)
+	decoder := expfmt.SampleDecoder{
+		Dec:  dec,
+		Opts: &expfmt.DecodeOptions{},
+	}
+
+	var samples []*model.Sample
+	for {
+		var v model.Vector
+		if err := decoder.Decode(&v); err != nil {
+			if err == io.EOF {
+				// Expected loop termination condition.
+				return samples, nil
+			}
+			return nil, err
+		}
+		samples = append(samples, v...)
+	}
+}
+
+// PodLatencyData encapsulates pod startup latency information.
+type PodLatencyData struct {
+	// Name of the pod
+	Name string
+	// Node this pod was running on
+	Node string
+	// Latency information related to pod startuptime
+	Latency time.Duration
+}
+
+type LatencySlice []PodLatencyData
+
+func (a LatencySlice) Len() int           { return len(a) }
+func (a LatencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a LatencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }
+
+func ExtractLatencyMetrics(latencies []PodLatencyData) LatencyMetric {
+	length := len(latencies)
+	perc50 := latencies[int(math.Ceil(float64(length*50)/100))-1].Latency
+	perc90 := latencies[int(math.Ceil(float64(length*90)/100))-1].Latency
+	perc99 := latencies[int(math.Ceil(float64(length*99)/100))-1].Latency
+	perc100 := latencies[length-1].Latency
+	return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99, Perc100: perc100}
+}
+
+// LogSuspiciousLatency logs metrics/docker errors from all nodes that had slow startup times
+// If latencyDataLag is nil then it will be populated from latencyData
+func LogSuspiciousLatency(latencyData []PodLatencyData, latencyDataLag []PodLatencyData, nodeCount int, c clientset.Interface) {
+	if latencyDataLag == nil {
+		latencyDataLag = latencyData
+	}
+	for _, l := range latencyData {
+		if l.Latency > NodeStartupThreshold {
+			HighLatencyKubeletOperations(c, 1*time.Second, l.Node, Logf)
+		}
+	}
+	Logf("Approx throughput: %v pods/min",
+		float64(nodeCount)/(latencyDataLag[len(latencyDataLag)-1].Latency.Minutes()))
+}
+
+func PrintLatencies(latencies []PodLatencyData, header string) {
+	metrics := ExtractLatencyMetrics(latencies)
+	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:])
+	Logf("perc50: %v, perc90: %v, perc99: %v", metrics.Perc50, metrics.Perc90, metrics.Perc99)
+}
+
+func (m *MetricsForE2E) computeClusterAutoscalerMetricsDelta(before metrics.Collection) {
+	if beforeSamples, found := before.ClusterAutoscalerMetrics[caFunctionMetric]; found {
+		if afterSamples, found := m.ClusterAutoscalerMetrics[caFunctionMetric]; found {
+			beforeSamplesMap := make(map[string]*model.Sample)
+			for _, bSample := range beforeSamples {
+				beforeSamplesMap[makeKey(bSample.Metric[caFunctionMetricLabel], bSample.Metric["le"])] = bSample
+			}
+			for _, aSample := range afterSamples {
+				if bSample, found := beforeSamplesMap[makeKey(aSample.Metric[caFunctionMetricLabel], aSample.Metric["le"])]; found {
+					aSample.Value = aSample.Value - bSample.Value
+				}
+
+			}
+		}
+	}
+}
+
+func makeKey(a, b model.LabelValue) string {
+	return string(a) + "___" + string(b)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/nodes_util.go

@@ -0,0 +1,400 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+func EtcdUpgrade(target_storage, target_version string) error {
+	switch TestContext.Provider {
+	case "gce":
+		return etcdUpgradeGCE(target_storage, target_version)
+	default:
+		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
+func IngressUpgrade(isUpgrade bool) error {
+	switch TestContext.Provider {
+	case "gce":
+		return ingressUpgradeGCE(isUpgrade)
+	default:
+		return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
+func MasterUpgrade(v string) error {
+	switch TestContext.Provider {
+	case "gce":
+		return masterUpgradeGCE(v, false)
+	case "gke":
+		return masterUpgradeGKE(v)
+	case "kubernetes-anywhere":
+		return masterUpgradeKubernetesAnywhere(v)
+	default:
+		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
+func etcdUpgradeGCE(target_storage, target_version string) error {
+	env := append(
+		os.Environ(),
+		"TEST_ETCD_VERSION="+target_version,
+		"STORAGE_BACKEND="+target_storage,
+		"TEST_ETCD_IMAGE=3.3.10-0")
+
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
+	return err
+}
+
+func ingressUpgradeGCE(isUpgrade bool) error {
+	var command string
+	if isUpgrade {
+		// User specified image to upgrade to.
+		targetImage := TestContext.IngressUpgradeImage
+		if targetImage != "" {
+			command = fmt.Sprintf("sudo sed -i -re 's|(image:)(.*)|\\1 %s|' /etc/kubernetes/manifests/glbc.manifest", targetImage)
+		} else {
+			// Upgrade to latest HEAD image.
+			command = "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/k8s-ingress-image-push\\/ingress-gce-e2e-glbc-amd64:master/' /etc/kubernetes/manifests/glbc.manifest"
+		}
+	} else {
+		// Downgrade to latest release image.
+		command = "sudo sed -i -re 's/(image:)(.*)/\\1 k8s.gcr.io\\/ingress-gce-glbc-amd64:v1.1.1/' /etc/kubernetes/manifests/glbc.manifest"
+	}
+	// Kubelet should restart glbc automatically.
+	sshResult, err := NodeExec(GetMasterHost(), command)
+	LogSSHResult(sshResult)
+	return err
+}
+
+// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
+func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
+	return masterUpgradeGCE(v, enableKubeProxyDaemonSet)
+}
+
+// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
+func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error {
+	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
+	// TODO: Remove these variables when they're no longer needed for downgrades.
+	if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" {
+		env = append(env,
+			"TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion,
+			"STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage,
+			"TEST_ETCD_IMAGE=3.3.10-0")
+	} else {
+		// In e2e tests, we skip the confirmation prompt about
+		// implicit etcd upgrades to simulate the user entering "y".
+		env = append(env, "TEST_ALLOW_IMPLICIT_ETCD_UPGRADE=true")
+	}
+
+	v := "v" + rawV
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-M", v)
+	return err
+}
+
+func locationParamGKE() string {
+	if TestContext.CloudConfig.MultiMaster {
+		// GKE Regional Clusters are being tested.
+		return fmt.Sprintf("--region=%s", TestContext.CloudConfig.Region)
+	}
+	return fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone)
+}
+
+func appendContainerCommandGroupIfNeeded(args []string) []string {
+	if TestContext.CloudConfig.Region != "" {
+		// TODO(wojtek-t): Get rid of it once Regional Clusters go to GA.
+		return append([]string{"beta"}, args...)
+	}
+	return args
+}
+
+func masterUpgradeGKE(v string) error {
+	Logf("Upgrading master to %q", v)
+	args := []string{
+		"container",
+		"clusters",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		locationParamGKE(),
+		"upgrade",
+		TestContext.CloudConfig.Cluster,
+		"--master",
+		fmt.Sprintf("--cluster-version=%s", v),
+		"--quiet",
+	}
+	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)
+	if err != nil {
+		return err
+	}
+
+	waitForSSHTunnels()
+
+	return nil
+}
+
+func masterUpgradeKubernetesAnywhere(v string) error {
+	Logf("Upgrading master to %q", v)
+
+	kaPath := TestContext.KubernetesAnywherePath
+	originalConfigPath := filepath.Join(kaPath, ".config")
+	backupConfigPath := filepath.Join(kaPath, ".config.bak")
+	updatedConfigPath := filepath.Join(kaPath, fmt.Sprintf(".config-%s", v))
+
+	// modify config with specified k8s version
+	if _, _, err := RunCmd("sed",
+		"-i.bak", // writes original to .config.bak
+		fmt.Sprintf(`s/kubernetes_version=.*$/kubernetes_version=%q/`, v),
+		originalConfigPath); err != nil {
+		return err
+	}
+
+	defer func() {
+		// revert .config.bak to .config
+		if err := os.Rename(backupConfigPath, originalConfigPath); err != nil {
+			Logf("Could not rename %s back to %s", backupConfigPath, originalConfigPath)
+		}
+	}()
+
+	// invoke ka upgrade
+	if _, _, err := RunCmd("make", "-C", TestContext.KubernetesAnywherePath,
+		"WAIT_FOR_KUBECONFIG=y", "upgrade-master"); err != nil {
+		return err
+	}
+
+	// move .config to .config.<version>
+	if err := os.Rename(originalConfigPath, updatedConfigPath); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func NodeUpgrade(f *Framework, v string, img string) error {
+	// Perform the upgrade.
+	var err error
+	switch TestContext.Provider {
+	case "gce":
+		err = nodeUpgradeGCE(v, img, false)
+	case "gke":
+		err = nodeUpgradeGKE(v, img)
+	default:
+		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+	if err != nil {
+		return err
+	}
+	return waitForNodesReadyAfterUpgrade(f)
+}
+
+// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
+func NodeUpgradeGCEWithKubeProxyDaemonSet(f *Framework, v string, img string, enableKubeProxyDaemonSet bool) error {
+	// Perform the upgrade.
+	if err := nodeUpgradeGCE(v, img, enableKubeProxyDaemonSet); err != nil {
+		return err
+	}
+	return waitForNodesReadyAfterUpgrade(f)
+}
+
+func waitForNodesReadyAfterUpgrade(f *Framework) error {
+	// Wait for it to complete and validate nodes are healthy.
+	//
+	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
+	// GKE; the operation shouldn't return until they all are.
+	numNodes, err := NumberOfRegisteredNodes(f.ClientSet)
+	if err != nil {
+		return fmt.Errorf("couldn't detect number of nodes")
+	}
+	Logf("Waiting up to %v for all %d nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout, numNodes)
+	if _, err := CheckNodesReady(f.ClientSet, numNodes, RestartNodeReadyAgainTimeout); err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
+func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error {
+	v := "v" + rawV
+	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
+	if img != "" {
+		env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img)
+		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
+		return err
+	}
+	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", v)
+	return err
+}
+
+func nodeUpgradeGKE(v string, img string) error {
+	Logf("Upgrading nodes to version %q and image %q", v, img)
+	args := []string{
+		"container",
+		"clusters",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		locationParamGKE(),
+		"upgrade",
+		TestContext.CloudConfig.Cluster,
+		fmt.Sprintf("--cluster-version=%s", v),
+		"--quiet",
+	}
+	if len(img) > 0 {
+		args = append(args, fmt.Sprintf("--image-type=%s", img))
+	}
+	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)
+
+	if err != nil {
+		return err
+	}
+
+	waitForSSHTunnels()
+
+	return nil
+}
+
+// MigTemplate (GCE-only) returns the name of the MIG template that the
+// nodes of the cluster use.
+func MigTemplate() (string, error) {
+	var errLast error
+	var templ string
+	key := "instanceTemplate"
+	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
+		// TODO(mikedanese): make this hit the compute API directly instead of
+		// shelling out to gcloud.
+		// An `instance-groups managed describe` call outputs what we want to stdout.
+		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
+			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+			"describe",
+			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
+			TestContext.CloudConfig.NodeInstanceGroup)
+		if err != nil {
+			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
+			return false, nil
+		}
+
+		// The 'describe' call probably succeeded; parse the output and try to
+		// find the line that looks like "instanceTemplate: url/to/<templ>" and
+		// return <templ>.
+		if val := ParseKVLines(output, key); len(val) > 0 {
+			url := strings.Split(val, "/")
+			templ = url[len(url)-1]
+			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
+			return true, nil
+		}
+		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
+		return false, nil
+	}) != nil {
+		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
+	}
+	return templ, nil
+}
+
+func gceUpgradeScript() string {
+	if len(TestContext.GCEUpgradeScript) == 0 {
+		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
+	}
+	return TestContext.GCEUpgradeScript
+}
+
+func waitForSSHTunnels() {
+	Logf("Waiting for SSH tunnels to establish")
+	RunKubectl("run", "ssh-tunnel-test",
+		"--image=busybox",
+		"--restart=Never",
+		"--command", "--",
+		"echo", "Hello")
+	defer RunKubectl("delete", "pod", "ssh-tunnel-test")
+
+	// allow up to a minute for new ssh tunnels to establish
+	wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
+		_, err := RunKubectl("logs", "ssh-tunnel-test")
+		return err == nil, nil
+	})
+}
+
+// NodeKiller is a utility to simulate node failures.
+type NodeKiller struct {
+	config   NodeKillerConfig
+	client   clientset.Interface
+	provider string
+}
+
+// NewNodeKiller creates new NodeKiller.
+func NewNodeKiller(config NodeKillerConfig, client clientset.Interface, provider string) *NodeKiller {
+	return &NodeKiller{config, client, provider}
+}
+
+// Run starts NodeKiller until stopCh is closed.
+func (k *NodeKiller) Run(stopCh <-chan struct{}) {
+	// wait.JitterUntil starts work immediately, so wait first.
+	time.Sleep(wait.Jitter(k.config.Interval, k.config.JitterFactor))
+	wait.JitterUntil(func() {
+		nodes := k.pickNodes()
+		k.kill(nodes)
+	}, k.config.Interval, k.config.JitterFactor, true, stopCh)
+}
+
+func (k *NodeKiller) pickNodes() []v1.Node {
+	nodes := GetReadySchedulableNodesOrDie(k.client)
+	numNodes := int(k.config.FailureRatio * float64(len(nodes.Items)))
+	shuffledNodes := shuffleNodes(nodes.Items)
+	if len(shuffledNodes) > numNodes {
+		return shuffledNodes[:numNodes]
+	}
+	return shuffledNodes
+}
+
+func (k *NodeKiller) kill(nodes []v1.Node) {
+	wg := sync.WaitGroup{}
+	wg.Add(len(nodes))
+	for _, node := range nodes {
+		node := node
+		go func() {
+			defer wg.Done()
+
+			Logf("Stopping docker and kubelet on %q to simulate failure", node.Name)
+			err := IssueSSHCommand("sudo systemctl stop docker kubelet", k.provider, &node)
+			if err != nil {
+				Logf("ERROR while stopping node %q: %v", node.Name, err)
+				return
+			}
+
+			time.Sleep(k.config.SimulatedDowntime)
+
+			Logf("Rebooting %q to repair the node", node.Name)
+			err = IssueSSHCommand("sudo reboot", k.provider, &node)
+			if err != nil {
+				Logf("ERROR while rebooting node %q: %v", node.Name, err)
+				return
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func DeleteNodeOnCloudProvider(node *v1.Node) error {
+	return TestContext.CloudConfig.Provider.DeleteNode(node)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/perf_util.go

@@ -0,0 +1,165 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/test/e2e/perftype"
+)
+
+// TODO(random-liu): Change the tests to actually use PerfData from the beginning instead of
+// translating one to the other here.
+
+// currentApiCallMetricsVersion is the current apicall performance metrics version. We should
+// bump up the version each time we make incompatible change to the metrics.
+const currentApiCallMetricsVersion = "v1"
+
+// ApiCallToPerfData transforms APIResponsiveness to PerfData.
+func ApiCallToPerfData(apicalls *APIResponsiveness) *perftype.PerfData {
+	perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
+	for _, apicall := range apicalls.APICalls {
+		item := perftype.DataItem{
+			Data: map[string]float64{
+				"Perc50": float64(apicall.Latency.Perc50) / 1000000, // us -> ms
+				"Perc90": float64(apicall.Latency.Perc90) / 1000000,
+				"Perc99": float64(apicall.Latency.Perc99) / 1000000,
+			},
+			Unit: "ms",
+			Labels: map[string]string{
+				"Verb":        apicall.Verb,
+				"Resource":    apicall.Resource,
+				"Subresource": apicall.Subresource,
+				"Scope":       apicall.Scope,
+				"Count":       fmt.Sprintf("%v", apicall.Count),
+			},
+		}
+		perfData.DataItems = append(perfData.DataItems, item)
+	}
+	return perfData
+}
+
+func latencyToPerfData(l LatencyMetric, name string) perftype.DataItem {
+	return perftype.DataItem{
+		Data: map[string]float64{
+			"Perc50":  float64(l.Perc50) / 1000000, // us -> ms
+			"Perc90":  float64(l.Perc90) / 1000000,
+			"Perc99":  float64(l.Perc99) / 1000000,
+			"Perc100": float64(l.Perc100) / 1000000,
+		},
+		Unit: "ms",
+		Labels: map[string]string{
+			"Metric": name,
+		},
+	}
+}
+
+// PodStartupLatencyToPerfData transforms PodStartupLatency to PerfData.
+func PodStartupLatencyToPerfData(latency *PodStartupLatency) *perftype.PerfData {
+	perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
+	perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.CreateToScheduleLatency, "create_to_schedule"))
+	perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToRunLatency, "schedule_to_run"))
+	perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.RunToWatchLatency, "run_to_watch"))
+	perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.ScheduleToWatchLatency, "schedule_to_watch"))
+	perfData.DataItems = append(perfData.DataItems, latencyToPerfData(latency.E2ELatency, "pod_startup"))
+	return perfData
+}
+
+// CurrentKubeletPerfMetricsVersion is the current kubelet performance metrics
+// version. This is used by mutiple perf related data structures. We should
+// bump up the version each time we make an incompatible change to the metrics.
+const CurrentKubeletPerfMetricsVersion = "v2"
+
+// ResourceUsageToPerfData transforms ResourceUsagePerNode to PerfData. Notice that this function
+// only cares about memory usage, because cpu usage information will be extracted from NodesCPUSummary.
+func ResourceUsageToPerfData(usagePerNode ResourceUsagePerNode) *perftype.PerfData {
+	return ResourceUsageToPerfDataWithLabels(usagePerNode, nil)
+}
+
+// CPUUsageToPerfData transforms NodesCPUSummary to PerfData.
+func CPUUsageToPerfData(usagePerNode NodesCPUSummary) *perftype.PerfData {
+	return CPUUsageToPerfDataWithLabels(usagePerNode, nil)
+}
+
+// PrintPerfData prints the perfdata in json format with PerfResultTag prefix.
+// If an error occurs, nothing will be printed.
+func PrintPerfData(p *perftype.PerfData) {
+	// Notice that we must make sure the perftype.PerfResultEnd is in a new line.
+	if str := PrettyPrintJSON(p); str != "" {
+		Logf("%s %s\n%s", perftype.PerfResultTag, str, perftype.PerfResultEnd)
+	}
+}
+
+// ResourceUsageToPerfDataWithLabels transforms ResourceUsagePerNode to PerfData with additional labels.
+// Notice that this function only cares about memory usage, because cpu usage information will be extracted from NodesCPUSummary.
+func ResourceUsageToPerfDataWithLabels(usagePerNode ResourceUsagePerNode, labels map[string]string) *perftype.PerfData {
+	items := []perftype.DataItem{}
+	for node, usages := range usagePerNode {
+		for c, usage := range usages {
+			item := perftype.DataItem{
+				Data: map[string]float64{
+					"memory":     float64(usage.MemoryUsageInBytes) / (1024 * 1024),
+					"workingset": float64(usage.MemoryWorkingSetInBytes) / (1024 * 1024),
+					"rss":        float64(usage.MemoryRSSInBytes) / (1024 * 1024),
+				},
+				Unit: "MB",
+				Labels: map[string]string{
+					"node":      node,
+					"container": c,
+					"datatype":  "resource",
+					"resource":  "memory",
+				},
+			}
+			items = append(items, item)
+		}
+	}
+	return &perftype.PerfData{
+		Version:   CurrentKubeletPerfMetricsVersion,
+		DataItems: items,
+		Labels:    labels,
+	}
+}
+
+// CPUUsageToPerfDataWithLabels transforms NodesCPUSummary to PerfData with additional labels.
+func CPUUsageToPerfDataWithLabels(usagePerNode NodesCPUSummary, labels map[string]string) *perftype.PerfData {
+	items := []perftype.DataItem{}
+	for node, usages := range usagePerNode {
+		for c, usage := range usages {
+			data := map[string]float64{}
+			for perc, value := range usage {
+				data[fmt.Sprintf("Perc%02.0f", perc*100)] = value * 1000
+			}
+
+			item := perftype.DataItem{
+				Data: data,
+				Unit: "mCPU",
+				Labels: map[string]string{
+					"node":      node,
+					"container": c,
+					"datatype":  "resource",
+					"resource":  "cpu",
+				},
+			}
+			items = append(items, item)
+		}
+	}
+	return &perftype.PerfData{
+		Version:   CurrentKubeletPerfMetricsVersion,
+		DataItems: items,
+		Labels:    labels,
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/pods.go

@@ -0,0 +1,294 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"regexp"
+	"sync"
+	"time"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+	"k8s.io/kubernetes/pkg/kubelet/events"
+	"k8s.io/kubernetes/pkg/kubelet/sysctl"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const DefaultPodDeletionTimeout = 3 * time.Minute
+
+// ImageWhiteList is the images used in the current test suite. It should be initialized in test suite and
+// the images in the white list should be pre-pulled in the test suite.  Currently, this is only used by
+// node e2e test.
+var ImageWhiteList sets.String
+
+// Convenience method for getting a pod client interface in the framework's namespace,
+// possibly applying test-suite specific transformations to the pod spec, e.g. for
+// node e2e pod scheduling.
+func (f *Framework) PodClient() *PodClient {
+	return &PodClient{
+		f:            f,
+		PodInterface: f.ClientSet.CoreV1().Pods(f.Namespace.Name),
+	}
+}
+
+// Convenience method for getting a pod client interface in an alternative namespace,
+// possibly applying test-suite specific transformations to the pod spec, e.g. for
+// node e2e pod scheduling.
+func (f *Framework) PodClientNS(namespace string) *PodClient {
+	return &PodClient{
+		f:            f,
+		PodInterface: f.ClientSet.CoreV1().Pods(namespace),
+	}
+}
+
+type PodClient struct {
+	f *Framework
+	v1core.PodInterface
+}
+
+// Create creates a new pod according to the framework specifications (don't wait for it to start).
+func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
+	c.mungeSpec(pod)
+	p, err := c.PodInterface.Create(pod)
+	ExpectNoError(err, "Error creating Pod")
+	return p
+}
+
+// CreateEventually retries pod creation for a while before failing
+// the test with the most recent error. This mimicks the behavior
+// of a controller (like the one for DaemonSet) and is necessary
+// because pod creation can fail while its service account is still
+// getting provisioned
+// (https://github.com/kubernetes/kubernetes/issues/68776).
+//
+// Both the timeout and polling interval are configurable as optional
+// arguments:
+// - The first optional argument is the timeout.
+// - The second optional argument is the polling interval.
+//
+// Both intervals can either be specified as time.Duration, parsable
+// duration strings or as floats/integers. In the last case they are
+// interpreted as seconds.
+func (c *PodClient) CreateEventually(pod *v1.Pod, opts ...interface{}) *v1.Pod {
+	c.mungeSpec(pod)
+	var ret *v1.Pod
+	Eventually(func() error {
+		p, err := c.PodInterface.Create(pod)
+		ret = p
+		return err
+	}, opts...).ShouldNot(HaveOccurred(), "Failed to create %q pod", pod.GetName())
+	return ret
+}
+
+// CreateSync creates a new pod according to the framework specifications in the given namespace, and waits for it to start.
+func (c *PodClient) CreateSyncInNamespace(pod *v1.Pod, namespace string) *v1.Pod {
+	p := c.Create(pod)
+	ExpectNoError(WaitForPodNameRunningInNamespace(c.f.ClientSet, p.Name, namespace))
+	// Get the newest pod after it becomes running, some status may change after pod created, such as pod ip.
+	p, err := c.Get(p.Name, metav1.GetOptions{})
+	ExpectNoError(err)
+	return p
+}
+
+// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
+func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
+	return c.CreateSyncInNamespace(pod, c.f.Namespace.Name)
+}
+
+// CreateBatch create a batch of pods. All pods are created before waiting.
+func (c *PodClient) CreateBatch(pods []*v1.Pod) []*v1.Pod {
+	ps := make([]*v1.Pod, len(pods))
+	var wg sync.WaitGroup
+	for i, pod := range pods {
+		wg.Add(1)
+		go func(i int, pod *v1.Pod) {
+			defer wg.Done()
+			defer GinkgoRecover()
+			ps[i] = c.CreateSync(pod)
+		}(i, pod)
+	}
+	wg.Wait()
+	return ps
+}
+
+// Update updates the pod object. It retries if there is a conflict, throw out error if
+// there is any other errors. name is the pod name, updateFn is the function updating the
+// pod object.
+func (c *PodClient) Update(name string, updateFn func(pod *v1.Pod)) {
+	ExpectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
+		pod, err := c.PodInterface.Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, fmt.Errorf("failed to get pod %q: %v", name, err)
+		}
+		updateFn(pod)
+		_, err = c.PodInterface.Update(pod)
+		if err == nil {
+			Logf("Successfully updated pod %q", name)
+			return true, nil
+		}
+		if errors.IsConflict(err) {
+			Logf("Conflicting update to pod %q, re-get and re-update: %v", name, err)
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to update pod %q: %v", name, err)
+	}))
+}
+
+// DeleteSync deletes the pod and wait for the pod to disappear for `timeout`. If the pod doesn't
+// disappear before the timeout, it will fail the test.
+func (c *PodClient) DeleteSync(name string, options *metav1.DeleteOptions, timeout time.Duration) {
+	c.DeleteSyncInNamespace(name, c.f.Namespace.Name, options, timeout)
+}
+
+// DeleteSyncInNamespace deletes the pod from the namespace and wait for the pod to disappear for `timeout`. If the pod doesn't
+// disappear before the timeout, it will fail the test.
+func (c *PodClient) DeleteSyncInNamespace(name string, namespace string, options *metav1.DeleteOptions, timeout time.Duration) {
+	err := c.Delete(name, options)
+	if err != nil && !errors.IsNotFound(err) {
+		Failf("Failed to delete pod %q: %v", name, err)
+	}
+	Expect(WaitForPodToDisappear(c.f.ClientSet, namespace, name, labels.Everything(),
+		2*time.Second, timeout)).To(Succeed(), "wait for pod %q to disappear", name)
+}
+
+// mungeSpec apply test-suite specific transformations to the pod spec.
+func (c *PodClient) mungeSpec(pod *v1.Pod) {
+	if !TestContext.NodeE2E {
+		return
+	}
+
+	Expect(pod.Spec.NodeName).To(Or(BeZero(), Equal(TestContext.NodeName)), "Test misconfigured")
+	pod.Spec.NodeName = TestContext.NodeName
+	// Node e2e does not support the default DNSClusterFirst policy. Set
+	// the policy to DNSDefault, which is configured per node.
+	pod.Spec.DNSPolicy = v1.DNSDefault
+
+	// PrepullImages only works for node e2e now. For cluster e2e, image prepull is not enforced,
+	// we should not munge ImagePullPolicy for cluster e2e pods.
+	if !TestContext.PrepullImages {
+		return
+	}
+	// If prepull is enabled, munge the container spec to make sure the images are not pulled
+	// during the test.
+	for i := range pod.Spec.Containers {
+		c := &pod.Spec.Containers[i]
+		if c.ImagePullPolicy == v1.PullAlways {
+			// If the image pull policy is PullAlways, the image doesn't need to be in
+			// the white list or pre-pulled, because the image is expected to be pulled
+			// in the test anyway.
+			continue
+		}
+		// If the image policy is not PullAlways, the image must be in the white list and
+		// pre-pulled.
+		Expect(ImageWhiteList.Has(c.Image)).To(BeTrue(), "Image %q is not in the white list, consider adding it to CommonImageWhiteList in test/e2e/common/util.go or NodeImageWhiteList in test/e2e_node/image_list.go", c.Image)
+		// Do not pull images during the tests because the images in white list should have
+		// been prepulled.
+		c.ImagePullPolicy = v1.PullNever
+	}
+}
+
+// TODO(random-liu): Move pod wait function into this file
+// WaitForSuccess waits for pod to succeed.
+func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
+	f := c.f
+	Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
+		func(pod *v1.Pod) (bool, error) {
+			switch pod.Status.Phase {
+			case v1.PodFailed:
+				return true, fmt.Errorf("pod %q failed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
+			case v1.PodSucceeded:
+				return true, nil
+			default:
+				return false, nil
+			}
+		},
+	)).To(Succeed(), "wait for pod %q to success", name)
+}
+
+// WaitForFailure waits for pod to fail.
+func (c *PodClient) WaitForFailure(name string, timeout time.Duration) {
+	f := c.f
+	Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
+		func(pod *v1.Pod) (bool, error) {
+			switch pod.Status.Phase {
+			case v1.PodFailed:
+				return true, nil
+			case v1.PodSucceeded:
+				return true, fmt.Errorf("pod %q successed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
+			default:
+				return false, nil
+			}
+		},
+	)).To(Succeed(), "wait for pod %q to fail", name)
+}
+
+// WaitForSuccess waits for pod to succeed or an error event for that pod.
+func (c *PodClient) WaitForErrorEventOrSuccess(pod *v1.Pod) (*v1.Event, error) {
+	var ev *v1.Event
+	err := wait.Poll(Poll, PodStartTimeout, func() (bool, error) {
+		evnts, err := c.f.ClientSet.CoreV1().Events(pod.Namespace).Search(legacyscheme.Scheme, pod)
+		if err != nil {
+			return false, fmt.Errorf("error in listing events: %s", err)
+		}
+		for _, e := range evnts.Items {
+			switch e.Reason {
+			case events.KillingContainer, events.FailedToCreateContainer, sysctl.UnsupportedReason, sysctl.ForbiddenReason:
+				ev = &e
+				return true, nil
+			case events.StartedContainer:
+				return true, nil
+			default:
+				// ignore all other errors
+			}
+		}
+		return false, nil
+	})
+	return ev, err
+}
+
+// MatchContainerOutput gets output of a container and match expected regexp in the output.
+func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
+	f := c.f
+	output, err := GetPodLogs(f.ClientSet, f.Namespace.Name, name, containerName)
+	if err != nil {
+		return fmt.Errorf("failed to get output for container %q of pod %q", containerName, name)
+	}
+	regex, err := regexp.Compile(expectedRegexp)
+	if err != nil {
+		return fmt.Errorf("failed to compile regexp %q: %v", expectedRegexp, err)
+	}
+	if !regex.MatchString(output) {
+		return fmt.Errorf("failed to match regexp %q in output %q", expectedRegexp, output)
+	}
+	return nil
+}
+
+func (c *PodClient) PodIsReady(name string) bool {
+	pod, err := c.Get(name, metav1.GetOptions{})
+	ExpectNoError(err)
+	return podutil.IsPodReady(pod)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/profile_gatherer.go

@@ -0,0 +1,209 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	// Default value for how long the CPU profile is gathered for.
+	DefaultCPUProfileSeconds = 30
+)
+
+func getProfilesDirectoryPath() string {
+	return path.Join(TestContext.ReportDir, "profiles")
+}
+
+func createProfilesDirectoryIfNeeded() error {
+	profileDirPath := getProfilesDirectoryPath()
+	if _, err := os.Stat(profileDirPath); os.IsNotExist(err) {
+		if mkdirErr := os.Mkdir(profileDirPath, 0777); mkdirErr != nil {
+			return fmt.Errorf("Failed to create profiles dir: %v", mkdirErr)
+		}
+	} else if err != nil {
+		return fmt.Errorf("Failed to check existence of profiles dir: %v", err)
+	}
+	return nil
+}
+
+func checkProfileGatheringPrerequisites() error {
+	if !TestContext.AllowGatheringProfiles {
+		return fmt.Errorf("Can't gather profiles as --allow-gathering-profiles is false")
+	}
+	if TestContext.ReportDir == "" {
+		return fmt.Errorf("Can't gather profiles as --report-dir is empty")
+	}
+	if err := createProfilesDirectoryIfNeeded(); err != nil {
+		return fmt.Errorf("Failed to ensure profiles dir: %v", err)
+	}
+	return nil
+}
+
+func getPortForComponent(componentName string) (int, error) {
+	switch componentName {
+	case "kube-apiserver":
+		return 8080, nil
+	case "kube-scheduler":
+		return 10251, nil
+	case "kube-controller-manager":
+		return 10252, nil
+	}
+	return -1, fmt.Errorf("Port for component %v unknown", componentName)
+}
+
+// Gathers profiles from a master component through SSH. E.g usages:
+//   - gatherProfile("kube-apiserver", "someTest", "heap")
+//   - gatherProfile("kube-scheduler", "someTest", "profile")
+//   - gatherProfile("kube-controller-manager", "someTest", "profile?seconds=20")
+//
+// We don't export this method but wrappers around it (see below).
+func gatherProfile(componentName, profileBaseName, profileKind string) error {
+	if err := checkProfileGatheringPrerequisites(); err != nil {
+		return fmt.Errorf("Profile gathering pre-requisite failed: %v", err)
+	}
+	profilePort, err := getPortForComponent(componentName)
+	if err != nil {
+		return fmt.Errorf("Profile gathering failed finding component port: %v", err)
+	}
+	if profileBaseName == "" {
+		profileBaseName = time.Now().Format(time.RFC3339)
+	}
+
+	// Get the profile data over SSH.
+	getCommand := fmt.Sprintf("curl -s localhost:%v/debug/pprof/%s", profilePort, profileKind)
+	sshResult, err := SSH(getCommand, GetMasterHost()+":22", TestContext.Provider)
+	if err != nil {
+		return fmt.Errorf("Failed to execute curl command on master through SSH: %v", err)
+	}
+
+	profilePrefix := componentName
+	switch {
+	case profileKind == "heap":
+		profilePrefix += "_MemoryProfile_"
+	case strings.HasPrefix(profileKind, "profile"):
+		profilePrefix += "_CPUProfile_"
+	default:
+		return fmt.Errorf("Unknown profile kind provided: %s", profileKind)
+	}
+
+	// Write the profile data to a file.
+	rawprofilePath := path.Join(getProfilesDirectoryPath(), profilePrefix+profileBaseName+".pprof")
+	rawprofile, err := os.Create(rawprofilePath)
+	if err != nil {
+		return fmt.Errorf("Failed to create file for the profile graph: %v", err)
+	}
+	defer rawprofile.Close()
+
+	if _, err := rawprofile.Write([]byte(sshResult.Stdout)); err != nil {
+		return fmt.Errorf("Failed to write file with profile data: %v", err)
+	}
+	if err := rawprofile.Close(); err != nil {
+		return fmt.Errorf("Failed to close file: %v", err)
+	}
+	// Create a graph from the data and write it to a pdf file.
+	var cmd *exec.Cmd
+	switch {
+	// TODO: Support other profile kinds if needed (e.g inuse_space, alloc_objects, mutex, etc)
+	case profileKind == "heap":
+		cmd = exec.Command("go", "tool", "pprof", "-pdf", "-symbolize=none", "--alloc_space", rawprofile.Name())
+	case strings.HasPrefix(profileKind, "profile"):
+		cmd = exec.Command("go", "tool", "pprof", "-pdf", "-symbolize=none", rawprofile.Name())
+	default:
+		return fmt.Errorf("Unknown profile kind provided: %s", profileKind)
+	}
+	outfilePath := path.Join(getProfilesDirectoryPath(), profilePrefix+profileBaseName+".pdf")
+	outfile, err := os.Create(outfilePath)
+	if err != nil {
+		return fmt.Errorf("Failed to create file for the profile graph: %v", err)
+	}
+	defer outfile.Close()
+	cmd.Stdout = outfile
+	stderr := bytes.NewBuffer(nil)
+	cmd.Stderr = stderr
+	if err := cmd.Run(); nil != err {
+		return fmt.Errorf("Failed to run 'go tool pprof': %v, stderr: %#v", err, stderr.String())
+	}
+	return nil
+}
+
+// The below exposed functions can take a while to execute as they SSH to the master,
+// collect and copy the profile over and then graph it. To allow waiting for these to
+// finish before the parent goroutine itself finishes, we accept a sync.WaitGroup
+// argument in these functions. Typically you would use the following pattern:
+//
+// func TestFoo() {
+//		var wg sync.WaitGroup
+//		wg.Add(3)
+//		go framework.GatherCPUProfile("kube-apiserver", "before_foo", &wg)
+//		go framework.GatherMemoryProfile("kube-apiserver", "before_foo", &wg)
+//		<<<< some code doing foo >>>>>>
+//		go framework.GatherCPUProfile("kube-scheduler", "after_foo", &wg)
+//		wg.Wait()
+// }
+//
+// If you do not wish to exercise the waiting logic, pass a nil value for the
+// waitgroup argument instead. However, then you would be responsible for ensuring
+// that the function finishes. There's also a polling-based gatherer utility for
+// CPU profiles available below.
+
+func GatherCPUProfile(componentName string, profileBaseName string, wg *sync.WaitGroup) {
+	GatherCPUProfileForSeconds(componentName, profileBaseName, DefaultCPUProfileSeconds, wg)
+}
+
+func GatherCPUProfileForSeconds(componentName string, profileBaseName string, seconds int, wg *sync.WaitGroup) {
+	if wg != nil {
+		defer wg.Done()
+	}
+	if err := gatherProfile(componentName, profileBaseName, fmt.Sprintf("profile?seconds=%v", seconds)); err != nil {
+		Logf("Failed to gather %v CPU profile: %v", componentName, err)
+	}
+}
+
+func GatherMemoryProfile(componentName string, profileBaseName string, wg *sync.WaitGroup) {
+	if wg != nil {
+		defer wg.Done()
+	}
+	if err := gatherProfile(componentName, profileBaseName, "heap"); err != nil {
+		Logf("Failed to gather %v memory profile: %v", componentName, err)
+	}
+}
+
+// StartCPUProfileGatherer performs polling-based gathering of the component's CPU
+// profile. It takes the interval b/w consecutive gatherings as an argument and
+// starts the gathering goroutine. To stop the gatherer, close the returned channel.
+func StartCPUProfileGatherer(componentName string, profileBaseName string, interval time.Duration) chan struct{} {
+	stopCh := make(chan struct{})
+	go func() {
+		for {
+			select {
+			case <-time.After(interval):
+				GatherCPUProfile(componentName, profileBaseName+"_"+time.Now().Format(time.RFC3339), nil)
+			case <-stopCh:
+				return
+			}
+		}
+	}()
+	return stopCh
+}

vendor/k8s.io/kubernetes/test/e2e/framework/provider.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+type Factory func() (ProviderInterface, error)
+
+var (
+	providers = make(map[string]Factory)
+	mutex     sync.Mutex
+)
+
+// RegisterProvider is expected to be called during application init,
+// typically by an init function in a provider package.
+func RegisterProvider(name string, factory Factory) {
+	mutex.Lock()
+	defer mutex.Unlock()
+	if _, ok := providers[name]; ok {
+		panic(fmt.Sprintf("provider %s already registered", name))
+	}
+	providers[name] = factory
+}
+
+// GetProviders returns the names of all currently registered providers.
+func GetProviders() []string {
+	mutex.Lock()
+	defer mutex.Unlock()
+	var providerNames []string
+	for name := range providers {
+		providerNames = append(providerNames, name)
+	}
+	return providerNames
+}
+
+func init() {
+	// "local" or "skeleton" can always be used.
+	RegisterProvider("local", func() (ProviderInterface, error) {
+		return NullProvider{}, nil
+	})
+	RegisterProvider("skeleton", func() (ProviderInterface, error) {
+		return NullProvider{}, nil
+	})
+	// The empty string used to be accepted in the past, but is not
+	// a valid value anymore.
+}
+
+// SetupProviderConfig validates the chosen provider and creates
+// an interface instance for it.
+func SetupProviderConfig(providerName string) (ProviderInterface, error) {
+	var err error
+
+	mutex.Lock()
+	defer mutex.Unlock()
+	factory, ok := providers[providerName]
+	if !ok {
+		return nil, errors.Wrapf(os.ErrNotExist, "The provider %s is unknown.", providerName)
+	}
+	provider, err := factory()
+
+	return provider, err
+}
+
+// ProviderInterface contains the implementation for certain
+// provider-specific functionality.
+type ProviderInterface interface {
+	FrameworkBeforeEach(f *Framework)
+	FrameworkAfterEach(f *Framework)
+
+	ResizeGroup(group string, size int32) error
+	GetGroupNodes(group string) ([]string, error)
+	GroupSize(group string) (int, error)
+
+	DeleteNode(node *v1.Node) error
+
+	CreatePD(zone string) (string, error)
+	DeletePD(pdName string) error
+	CreatePVSource(zone, diskName string) (*v1.PersistentVolumeSource, error)
+	DeletePVSource(pvSource *v1.PersistentVolumeSource) error
+
+	CleanupServiceResources(c clientset.Interface, loadBalancerName, region, zone string)
+
+	EnsureLoadBalancerResourcesDeleted(ip, portRange string) error
+	LoadBalancerSrcRanges() []string
+	EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service))
+}
+
+// NullProvider is the default implementation of the ProviderInterface
+// which doesn't do anything.
+type NullProvider struct{}
+
+func (n NullProvider) FrameworkBeforeEach(f *Framework) {}
+func (n NullProvider) FrameworkAfterEach(f *Framework)  {}
+
+func (n NullProvider) ResizeGroup(string, int32) error {
+	return fmt.Errorf("Provider does not support InstanceGroups")
+}
+func (n NullProvider) GetGroupNodes(group string) ([]string, error) {
+	return nil, fmt.Errorf("provider does not support InstanceGroups")
+}
+func (n NullProvider) GroupSize(group string) (int, error) {
+	return -1, fmt.Errorf("provider does not support InstanceGroups")
+}
+
+func (n NullProvider) DeleteNode(node *v1.Node) error {
+	return fmt.Errorf("provider does not support DeleteNode")
+}
+
+func (n NullProvider) CreatePD(zone string) (string, error) {
+	return "", fmt.Errorf("provider does not support volume creation")
+}
+func (n NullProvider) DeletePD(pdName string) error {
+	return fmt.Errorf("provider does not support volume deletion")
+}
+func (n NullProvider) CreatePVSource(zone, diskName string) (*v1.PersistentVolumeSource, error) {
+	return nil, fmt.Errorf("Provider not supported")
+}
+func (n NullProvider) DeletePVSource(pvSource *v1.PersistentVolumeSource) error {
+	return fmt.Errorf("Provider not supported")
+}
+
+func (n NullProvider) CleanupServiceResources(c clientset.Interface, loadBalancerName, region, zone string) {
+}
+
+func (n NullProvider) EnsureLoadBalancerResourcesDeleted(ip, portRange string) error {
+	return nil
+}
+func (n NullProvider) LoadBalancerSrcRanges() []string {
+	return nil
+}
+func (n NullProvider) EnableAndDisableInternalLB() (enable, disable func(svc *v1.Service)) {
+	nop := func(svc *v1.Service) {}
+	return nop, nop
+}
+
+var _ ProviderInterface = NullProvider{}

vendor/k8s.io/kubernetes/test/e2e/framework/psp_util.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"sync"
+
+	corev1 "k8s.io/api/core/v1"
+	policy "k8s.io/api/policy/v1beta1"
+	rbacv1beta1 "k8s.io/api/rbac/v1beta1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/kubernetes/pkg/security/podsecuritypolicy/seccomp"
+
+	. "github.com/onsi/ginkgo"
+)
+
+const (
+	podSecurityPolicyPrivileged = "e2e-test-privileged-psp"
+)
+
+var (
+	isPSPEnabledOnce sync.Once
+	isPSPEnabled     bool
+)
+
+// Creates a PodSecurityPolicy that allows everything.
+func PrivilegedPSP(name string) *policy.PodSecurityPolicy {
+	allowPrivilegeEscalation := true
+	return &policy.PodSecurityPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        name,
+			Annotations: map[string]string{seccomp.AllowedProfilesAnnotationKey: seccomp.AllowAny},
+		},
+		Spec: policy.PodSecurityPolicySpec{
+			Privileged:               true,
+			AllowPrivilegeEscalation: &allowPrivilegeEscalation,
+			AllowedCapabilities:      []corev1.Capability{"*"},
+			Volumes:                  []policy.FSType{policy.All},
+			HostNetwork:              true,
+			HostPorts:                []policy.HostPortRange{{Min: 0, Max: 65535}},
+			HostIPC:                  true,
+			HostPID:                  true,
+			RunAsUser: policy.RunAsUserStrategyOptions{
+				Rule: policy.RunAsUserStrategyRunAsAny,
+			},
+			SELinux: policy.SELinuxStrategyOptions{
+				Rule: policy.SELinuxStrategyRunAsAny,
+			},
+			SupplementalGroups: policy.SupplementalGroupsStrategyOptions{
+				Rule: policy.SupplementalGroupsStrategyRunAsAny,
+			},
+			FSGroup: policy.FSGroupStrategyOptions{
+				Rule: policy.FSGroupStrategyRunAsAny,
+			},
+			ReadOnlyRootFilesystem: false,
+			AllowedUnsafeSysctls:   []string{"*"},
+		},
+	}
+}
+
+func IsPodSecurityPolicyEnabled(f *Framework) bool {
+	isPSPEnabledOnce.Do(func() {
+		psps, err := f.ClientSet.PolicyV1beta1().PodSecurityPolicies().List(metav1.ListOptions{})
+		if err != nil {
+			Logf("Error listing PodSecurityPolicies; assuming PodSecurityPolicy is disabled: %v", err)
+			isPSPEnabled = false
+		} else if psps == nil || len(psps.Items) == 0 {
+			Logf("No PodSecurityPolicies found; assuming PodSecurityPolicy is disabled.")
+			isPSPEnabled = false
+		} else {
+			Logf("Found PodSecurityPolicies; assuming PodSecurityPolicy is enabled.")
+			isPSPEnabled = true
+		}
+	})
+	return isPSPEnabled
+}
+
+var (
+	privilegedPSPOnce sync.Once
+)
+
+func CreatePrivilegedPSPBinding(f *Framework, namespace string) {
+	if !IsPodSecurityPolicyEnabled(f) {
+		return
+	}
+	// Create the privileged PSP & role
+	privilegedPSPOnce.Do(func() {
+		_, err := f.ClientSet.PolicyV1beta1().PodSecurityPolicies().Get(
+			podSecurityPolicyPrivileged, metav1.GetOptions{})
+		if !apierrs.IsNotFound(err) {
+			// Privileged PSP was already created.
+			ExpectNoError(err, "Failed to get PodSecurityPolicy %s", podSecurityPolicyPrivileged)
+			return
+		}
+
+		psp := PrivilegedPSP(podSecurityPolicyPrivileged)
+		psp, err = f.ClientSet.PolicyV1beta1().PodSecurityPolicies().Create(psp)
+		if !apierrs.IsAlreadyExists(err) {
+			ExpectNoError(err, "Failed to create PSP %s", podSecurityPolicyPrivileged)
+		}
+
+		if IsRBACEnabled(f) {
+			// Create the Role to bind it to the namespace.
+			_, err = f.ClientSet.RbacV1beta1().ClusterRoles().Create(&rbacv1beta1.ClusterRole{
+				ObjectMeta: metav1.ObjectMeta{Name: podSecurityPolicyPrivileged},
+				Rules: []rbacv1beta1.PolicyRule{{
+					APIGroups:     []string{"extensions"},
+					Resources:     []string{"podsecuritypolicies"},
+					ResourceNames: []string{podSecurityPolicyPrivileged},
+					Verbs:         []string{"use"},
+				}},
+			})
+			if !apierrs.IsAlreadyExists(err) {
+				ExpectNoError(err, "Failed to create PSP role")
+			}
+		}
+	})
+
+	if IsRBACEnabled(f) {
+		By(fmt.Sprintf("Binding the %s PodSecurityPolicy to the default service account in %s",
+			podSecurityPolicyPrivileged, namespace))
+		BindClusterRoleInNamespace(f.ClientSet.RbacV1beta1(),
+			podSecurityPolicyPrivileged,
+			namespace,
+			rbacv1beta1.Subject{
+				Kind:      rbacv1beta1.ServiceAccountKind,
+				Namespace: namespace,
+				Name:      "default",
+			})
+		ExpectNoError(WaitForNamedAuthorizationUpdate(f.ClientSet.AuthorizationV1beta1(),
+			serviceaccount.MakeUsername(namespace, "default"), namespace, "use", podSecurityPolicyPrivileged,
+			schema.GroupResource{Group: "extensions", Resource: "podsecuritypolicies"}, true))
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/pv_util.go

@@ -0,0 +1,962 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/volume/util"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+const (
+	PDRetryTimeout    = 5 * time.Minute
+	PDRetryPollTime   = 5 * time.Second
+	VolumeSelectorKey = "e2e-pv-pool"
+)
+
+var (
+	// Common selinux labels
+	SELinuxLabel = &v1.SELinuxOptions{
+		Level: "s0:c0,c1"}
+)
+
+// Map of all PVs used in the multi pv-pvc tests. The key is the PV's name, which is
+// guaranteed to be unique. The value is {} (empty struct) since we're only interested
+// in the PV's name and if it is present. We must always Get the pv object before
+// referencing any of its values, eg its ClaimRef.
+type pvval struct{}
+type PVMap map[string]pvval
+
+// Map of all PVCs used in the multi pv-pvc tests. The key is "namespace/pvc.Name". The
+// value is {} (empty struct) since we're only interested in the PVC's name and if it is
+// present. We must always Get the pvc object before referencing any of its values, eg.
+// its VolumeName.
+// Note: It's unsafe to add keys to a map in a loop. Their insertion in the map is
+//   unpredictable and can result in the same key being iterated over again.
+type pvcval struct{}
+type PVCMap map[types.NamespacedName]pvcval
+
+// PersistentVolumeConfig is consumed by MakePersistentVolume() to generate a PV object
+// for varying storage options (NFS, ceph, glusterFS, etc.).
+// (+optional) prebind holds a pre-bound PVC
+// Example pvSource:
+//	pvSource: api.PersistentVolumeSource{
+//		NFS: &api.NFSVolumeSource{
+//	 		...
+//	 	},
+//	 }
+type PersistentVolumeConfig struct {
+	PVSource         v1.PersistentVolumeSource
+	Prebind          *v1.PersistentVolumeClaim
+	ReclaimPolicy    v1.PersistentVolumeReclaimPolicy
+	NamePrefix       string
+	Labels           labels.Set
+	StorageClassName string
+	NodeAffinity     *v1.VolumeNodeAffinity
+	VolumeMode       *v1.PersistentVolumeMode
+}
+
+// PersistentVolumeClaimConfig is consumed by MakePersistentVolumeClaim() to generate a PVC object.
+// AccessModes defaults to all modes (RWO, RWX, ROX) if left empty
+// (+optional) Annotations defines the PVC's annotations
+
+type PersistentVolumeClaimConfig struct {
+	AccessModes      []v1.PersistentVolumeAccessMode
+	Annotations      map[string]string
+	Selector         *metav1.LabelSelector
+	StorageClassName *string
+	VolumeMode       *v1.PersistentVolumeMode
+}
+
+// Clean up a pv and pvc in a single pv/pvc test case.
+// Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
+func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
+	var errs []error
+
+	if pvc != nil {
+		err := DeletePersistentVolumeClaim(c, pvc.Name, ns)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PVC %q: %v", pvc.Name, err))
+		}
+	} else {
+		Logf("pvc is nil")
+	}
+	if pv != nil {
+		err := DeletePersistentVolume(c, pv.Name)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PV %q: %v", pv.Name, err))
+		}
+	} else {
+		Logf("pv is nil")
+	}
+	return errs
+}
+
+// Clean up pvs and pvcs in multi-pv-pvc test cases. Entries found in the pv and claim maps are
+// deleted as long as the Delete api call succeeds.
+// Note: delete errors are appended to []error so that as many pvcs and pvs as possible are deleted.
+func PVPVCMapCleanup(c clientset.Interface, ns string, pvols PVMap, claims PVCMap) []error {
+	var errs []error
+
+	for pvcKey := range claims {
+		err := DeletePersistentVolumeClaim(c, pvcKey.Name, ns)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PVC %q: %v", pvcKey.Name, err))
+		} else {
+			delete(claims, pvcKey)
+		}
+	}
+
+	for pvKey := range pvols {
+		err := DeletePersistentVolume(c, pvKey)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PV %q: %v", pvKey, err))
+		} else {
+			delete(pvols, pvKey)
+		}
+	}
+	return errs
+}
+
+// Delete the PV.
+func DeletePersistentVolume(c clientset.Interface, pvName string) error {
+	if c != nil && len(pvName) > 0 {
+		Logf("Deleting PersistentVolume %q", pvName)
+		err := c.CoreV1().PersistentVolumes().Delete(pvName, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("PV Delete API error: %v", err)
+		}
+	}
+	return nil
+}
+
+// Delete the Claim
+func DeletePersistentVolumeClaim(c clientset.Interface, pvcName string, ns string) error {
+	if c != nil && len(pvcName) > 0 {
+		Logf("Deleting PersistentVolumeClaim %q", pvcName)
+		err := c.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("PVC Delete API error: %v", err)
+		}
+	}
+	return nil
+}
+
+// Delete the PVC and wait for the PV to enter its expected phase. Validate that the PV
+// has been reclaimed (assumption here about reclaimPolicy). Caller tells this func which
+// phase value to expect for the pv bound to the to-be-deleted claim.
+func DeletePVCandValidatePV(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, expectPVPhase v1.PersistentVolumePhase) error {
+	pvname := pvc.Spec.VolumeName
+	Logf("Deleting PVC %v to trigger reclamation of PV %v", pvc.Name, pvname)
+	err := DeletePersistentVolumeClaim(c, pvc.Name, ns)
+	if err != nil {
+		return err
+	}
+
+	// Wait for the PV's phase to return to be `expectPVPhase`
+	Logf("Waiting for reclaim process to complete.")
+	err = WaitForPersistentVolumePhase(expectPVPhase, c, pv.Name, Poll, PVReclaimingTimeout)
+	if err != nil {
+		return fmt.Errorf("pv %q phase did not become %v: %v", pv.Name, expectPVPhase, err)
+	}
+
+	// examine the pv's ClaimRef and UID and compare to expected values
+	pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PV Get API error: %v", err)
+	}
+	cr := pv.Spec.ClaimRef
+	if expectPVPhase == v1.VolumeAvailable {
+		if cr != nil && len(cr.UID) > 0 {
+			return fmt.Errorf("PV is 'Available' but ClaimRef.UID is not empty")
+		}
+	} else if expectPVPhase == v1.VolumeBound {
+		if cr == nil {
+			return fmt.Errorf("PV is 'Bound' but ClaimRef is nil")
+		}
+		if len(cr.UID) == 0 {
+			return fmt.Errorf("PV is 'Bound' but ClaimRef.UID is empty")
+		}
+	}
+
+	Logf("PV %v now in %q phase", pv.Name, expectPVPhase)
+	return nil
+}
+
+// Wraps deletePVCandValidatePV() by calling the function in a loop over the PV map. Only bound PVs
+// are deleted. Validates that the claim was deleted and the PV is in the expected Phase (Released,
+// Available, Bound).
+// Note: if there are more claims than pvs then some of the remaining claims may bind to just made
+//   available pvs.
+func DeletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols PVMap, claims PVCMap, expectPVPhase v1.PersistentVolumePhase) error {
+	var boundPVs, deletedPVCs int
+
+	for pvName := range pvols {
+		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("PV Get API error: %v", err)
+		}
+		cr := pv.Spec.ClaimRef
+		// if pv is bound then delete the pvc it is bound to
+		if cr != nil && len(cr.Name) > 0 {
+			boundPVs++
+			// Assert bound PVC is tracked in this test. Failing this might
+			// indicate external PVCs interfering with the test.
+			pvcKey := makePvcKey(ns, cr.Name)
+			if _, found := claims[pvcKey]; !found {
+				return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey)
+			}
+			// get the pvc for the delete call below
+			pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(cr.Name, metav1.GetOptions{})
+			if err == nil {
+				if err = DeletePVCandValidatePV(c, ns, pvc, pv, expectPVPhase); err != nil {
+					return err
+				}
+			} else if !apierrs.IsNotFound(err) {
+				return fmt.Errorf("PVC Get API error: %v", err)
+			}
+			// delete pvckey from map even if apierrs.IsNotFound above is true and thus the
+			// claim was not actually deleted here
+			delete(claims, pvcKey)
+			deletedPVCs++
+		}
+	}
+	if boundPVs != deletedPVCs {
+		return fmt.Errorf("expect number of bound PVs (%v) to equal number of deleted PVCs (%v)", boundPVs, deletedPVCs)
+	}
+	return nil
+}
+
+// create the PV resource. Fails test on error.
+func createPV(c clientset.Interface, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
+	pv, err := c.CoreV1().PersistentVolumes().Create(pv)
+	if err != nil {
+		return nil, fmt.Errorf("PV Create API error: %v", err)
+	}
+	return pv, nil
+}
+
+// create the PV resource. Fails test on error.
+func CreatePV(c clientset.Interface, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
+	return createPV(c, pv)
+}
+
+// create the PVC resource. Fails test on error.
+func CreatePVC(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolumeClaim, error) {
+	pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+	if err != nil {
+		return nil, fmt.Errorf("PVC Create API error: %v", err)
+	}
+	return pvc, nil
+}
+
+// Create a PVC followed by the PV based on the passed in nfs-server ip and
+// namespace. If the "preBind" bool is true then pre-bind the PV to the PVC
+// via the PV's ClaimRef. Return the pv and pvc to reflect the created objects.
+// Note: in the pre-bind case the real PVC name, which is generated, is not
+//   known until after the PVC is instantiated. This is why the pvc is created
+//   before the pv.
+func CreatePVCPV(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) {
+	// make the pvc spec
+	pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+	preBindMsg := ""
+	if preBind {
+		preBindMsg = " pre-bound"
+		pvConfig.Prebind = pvc
+	}
+	// make the pv spec
+	pv := MakePersistentVolume(pvConfig)
+
+	By(fmt.Sprintf("Creating a PVC followed by a%s PV", preBindMsg))
+	pvc, err := CreatePVC(c, ns, pvc)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// instantiate the pv, handle pre-binding by ClaimRef if needed
+	if preBind {
+		pv.Spec.ClaimRef.Name = pvc.Name
+	}
+	pv, err = createPV(c, pv)
+	if err != nil {
+		return nil, pvc, err
+	}
+	return pv, pvc, nil
+}
+
+// Create a PV followed by the PVC based on the passed in nfs-server ip and
+// namespace. If the "preBind" bool is true then pre-bind the PVC to the PV
+// via the PVC's VolumeName. Return the pv and pvc to reflect the created
+// objects.
+// Note: in the pre-bind case the real PV name, which is generated, is not
+//   known until after the PV is instantiated. This is why the pv is created
+//   before the pvc.
+func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) {
+	preBindMsg := ""
+	if preBind {
+		preBindMsg = " pre-bound"
+	}
+	Logf("Creating a PV followed by a%s PVC", preBindMsg)
+
+	// make the pv and pvc definitions
+	pv := MakePersistentVolume(pvConfig)
+	pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+
+	// instantiate the pv
+	pv, err := createPV(c, pv)
+	if err != nil {
+		return nil, nil, err
+	}
+	// instantiate the pvc, handle pre-binding by VolumeName if needed
+	if preBind {
+		pvc.Spec.VolumeName = pv.Name
+	}
+	pvc, err = CreatePVC(c, ns, pvc)
+	if err != nil {
+		return pv, nil, err
+	}
+	return pv, pvc, nil
+}
+
+// Create the desired number of PVs and PVCs and return them in separate maps. If the
+// number of PVs != the number of PVCs then the min of those two counts is the number of
+// PVs expected to bind. If a Create error occurs, the returned maps may contain pv and pvc
+// entries for the resources that were successfully created. In other words, when the caller
+// sees an error returned, it needs to decide what to do about entries in the maps.
+// Note: when the test suite deletes the namespace orphaned pvcs and pods are deleted. However,
+//   orphaned pvs are not deleted and will remain after the suite completes.
+func CreatePVsPVCs(numpvs, numpvcs int, c clientset.Interface, ns string, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig) (PVMap, PVCMap, error) {
+	pvMap := make(PVMap, numpvs)
+	pvcMap := make(PVCMap, numpvcs)
+	extraPVCs := 0
+	extraPVs := numpvs - numpvcs
+	if extraPVs < 0 {
+		extraPVCs = -extraPVs
+		extraPVs = 0
+	}
+	pvsToCreate := numpvs - extraPVs // want the min(numpvs, numpvcs)
+
+	// create pvs and pvcs
+	for i := 0; i < pvsToCreate; i++ {
+		pv, pvc, err := CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvMap[pv.Name] = pvval{}
+		pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
+	}
+
+	// create extra pvs or pvcs as needed
+	for i := 0; i < extraPVs; i++ {
+		pv := MakePersistentVolume(pvConfig)
+		pv, err := createPV(c, pv)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvMap[pv.Name] = pvval{}
+	}
+	for i := 0; i < extraPVCs; i++ {
+		pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+		pvc, err := CreatePVC(c, ns, pvc)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
+	}
+	return pvMap, pvcMap, nil
+}
+
+// Wait for the pv and pvc to bind to each other.
+func WaitOnPVandPVC(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) error {
+	// Wait for newly created PVC to bind to the PV
+	Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name)
+	err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, Poll, ClaimBindingTimeout)
+	if err != nil {
+		return fmt.Errorf("PVC %q did not become Bound: %v", pvc.Name, err)
+	}
+
+	// Wait for PersistentVolume.Status.Phase to be Bound, which it should be
+	// since the PVC is already bound.
+	err = WaitForPersistentVolumePhase(v1.VolumeBound, c, pv.Name, Poll, PVBindingTimeout)
+	if err != nil {
+		return fmt.Errorf("PV %q did not become Bound: %v", pv.Name, err)
+	}
+
+	// Re-get the pv and pvc objects
+	pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PV Get API error: %v", err)
+	}
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PVC Get API error: %v", err)
+	}
+
+	// The pv and pvc are both bound, but to each other?
+	// Check that the PersistentVolume.ClaimRef matches the PVC
+	if pv.Spec.ClaimRef == nil {
+		return fmt.Errorf("PV %q ClaimRef is nil", pv.Name)
+	}
+	if pv.Spec.ClaimRef.Name != pvc.Name {
+		return fmt.Errorf("PV %q ClaimRef's name (%q) should be %q", pv.Name, pv.Spec.ClaimRef.Name, pvc.Name)
+	}
+	if pvc.Spec.VolumeName != pv.Name {
+		return fmt.Errorf("PVC %q VolumeName (%q) should be %q", pvc.Name, pvc.Spec.VolumeName, pv.Name)
+	}
+	if pv.Spec.ClaimRef.UID != pvc.UID {
+		return fmt.Errorf("PV %q ClaimRef's UID (%q) should be %q", pv.Name, pv.Spec.ClaimRef.UID, pvc.UID)
+	}
+	return nil
+}
+
+// Search for bound PVs and PVCs by examining pvols for non-nil claimRefs.
+// NOTE: Each iteration waits for a maximum of 3 minutes per PV and, if the PV is bound,
+//   up to 3 minutes for the PVC. When the number of PVs != number of PVCs, this can lead
+//   to situations where the maximum wait times are reached several times in succession,
+//   extending test time. Thus, it is recommended to keep the delta between PVs and PVCs
+//   small.
+func WaitAndVerifyBinds(c clientset.Interface, ns string, pvols PVMap, claims PVCMap, testExpected bool) error {
+	var actualBinds int
+	expectedBinds := len(pvols)
+	if expectedBinds > len(claims) { // want the min of # pvs or #pvcs
+		expectedBinds = len(claims)
+	}
+
+	for pvName := range pvols {
+		err := WaitForPersistentVolumePhase(v1.VolumeBound, c, pvName, Poll, PVBindingTimeout)
+		if err != nil && len(pvols) > len(claims) {
+			Logf("WARN: pv %v is not bound after max wait", pvName)
+			Logf("      This may be ok since there are more pvs than pvcs")
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("PV %q did not become Bound: %v", pvName, err)
+		}
+
+		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("PV Get API error: %v", err)
+		}
+		cr := pv.Spec.ClaimRef
+		if cr != nil && len(cr.Name) > 0 {
+			// Assert bound pvc is a test resource. Failing assertion could
+			// indicate non-test PVC interference or a bug in the test
+			pvcKey := makePvcKey(ns, cr.Name)
+			if _, found := claims[pvcKey]; !found {
+				return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey)
+			}
+
+			err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, cr.Name, Poll, ClaimBindingTimeout)
+			if err != nil {
+				return fmt.Errorf("PVC %q did not become Bound: %v", cr.Name, err)
+			}
+			actualBinds++
+		}
+	}
+
+	if testExpected && actualBinds != expectedBinds {
+		return fmt.Errorf("expect number of bound PVs (%v) to equal number of claims (%v)", actualBinds, expectedBinds)
+	}
+	return nil
+}
+
+// Test the pod's exit code to be zero.
+func testPodSuccessOrFail(c clientset.Interface, ns string, pod *v1.Pod) error {
+	By("Pod should terminate with exitcode 0 (success)")
+	if err := WaitForPodSuccessInNamespace(c, pod.Name, ns); err != nil {
+		return fmt.Errorf("pod %q failed to reach Success: %v", pod.Name, err)
+	}
+	Logf("Pod %v succeeded ", pod.Name)
+	return nil
+}
+
+// Deletes the passed-in pod and waits for the pod to be terminated. Resilient to the pod
+// not existing.
+func DeletePodWithWait(f *Framework, c clientset.Interface, pod *v1.Pod) error {
+	if pod == nil {
+		return nil
+	}
+	return DeletePodWithWaitByName(f, c, pod.GetName(), pod.GetNamespace())
+}
+
+// Deletes the named and namespaced pod and waits for the pod to be terminated. Resilient to the pod
+// not existing.
+func DeletePodWithWaitByName(f *Framework, c clientset.Interface, podName, podNamespace string) error {
+	const maxWait = 5 * time.Minute
+	Logf("Deleting pod %q in namespace %q", podName, podNamespace)
+	err := c.CoreV1().Pods(podNamespace).Delete(podName, nil)
+	if err != nil {
+		if apierrs.IsNotFound(err) {
+			return nil // assume pod was already deleted
+		}
+		return fmt.Errorf("pod Delete API error: %v", err)
+	}
+	Logf("Wait up to %v for pod %q to be fully deleted", maxWait, podName)
+	err = f.WaitForPodNotFound(podName, maxWait)
+	if err != nil {
+		return fmt.Errorf("pod %q was not deleted: %v", podName, err)
+	}
+	return nil
+}
+
+// Create the test pod, wait for (hopefully) success, and then delete the pod.
+// Note: need named return value so that the err assignment in the defer sets the returned error.
+//       Has been shown to be necessary using Go 1.7.
+func CreateWaitAndDeletePod(f *Framework, c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (err error) {
+	Logf("Creating nfs test pod")
+	pod := MakeWritePod(ns, pvc)
+	runPod, err := c.CoreV1().Pods(ns).Create(pod)
+	if err != nil {
+		return fmt.Errorf("pod Create API error: %v", err)
+	}
+	defer func() {
+		delErr := DeletePodWithWait(f, c, runPod)
+		if err == nil { // don't override previous err value
+			err = delErr // assign to returned err, can be nil
+		}
+	}()
+
+	err = testPodSuccessOrFail(c, ns, runPod)
+	if err != nil {
+		return fmt.Errorf("pod %q did not exit with Success: %v", runPod.Name, err)
+	}
+	return // note: named return value
+}
+
+// Return a pvckey struct.
+func makePvcKey(ns, name string) types.NamespacedName {
+	return types.NamespacedName{Namespace: ns, Name: name}
+}
+
+// Returns a PV definition based on the nfs server IP. If the PVC is not nil
+// then the PV is defined with a ClaimRef which includes the PVC's namespace.
+// If the PVC is nil then the PV is not defined with a ClaimRef.  If no reclaimPolicy
+// is assigned, assumes "Retain". Specs are expected to match the test's PVC.
+// Note: the passed-in claim does not have a name until it is created and thus the PV's
+//   ClaimRef cannot be completely filled-in in this func. Therefore, the ClaimRef's name
+//   is added later in CreatePVCPV.
+func MakePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume {
+	var claimRef *v1.ObjectReference
+	// If the reclaimPolicy is not provided, assume Retain
+	if pvConfig.ReclaimPolicy == "" {
+		Logf("PV ReclaimPolicy unspecified, default: Retain")
+		pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRetain
+	}
+	if pvConfig.Prebind != nil {
+		claimRef = &v1.ObjectReference{
+			Name:      pvConfig.Prebind.Name,
+			Namespace: pvConfig.Prebind.Namespace,
+		}
+	}
+	return &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: pvConfig.NamePrefix,
+			Labels:       pvConfig.Labels,
+			Annotations: map[string]string{
+				util.VolumeGidAnnotationKey: "777",
+			},
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeReclaimPolicy: pvConfig.ReclaimPolicy,
+			Capacity: v1.ResourceList{
+				v1.ResourceName(v1.ResourceStorage): resource.MustParse("2Gi"),
+			},
+			PersistentVolumeSource: pvConfig.PVSource,
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+				v1.ReadOnlyMany,
+				v1.ReadWriteMany,
+			},
+			ClaimRef:         claimRef,
+			StorageClassName: pvConfig.StorageClassName,
+			NodeAffinity:     pvConfig.NodeAffinity,
+			VolumeMode:       pvConfig.VolumeMode,
+		},
+	}
+}
+
+// Returns a PVC definition based on the namespace.
+// Note: if this PVC is intended to be pre-bound to a PV, whose name is not
+//   known until the PV is instantiated, then the func CreatePVPVC will add
+//   pvc.Spec.VolumeName to this claim.
+func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.PersistentVolumeClaim {
+	// Specs are expected to match this test's PersistentVolume
+
+	if len(cfg.AccessModes) == 0 {
+		Logf("AccessModes unspecified, default: all modes (RWO, RWX, ROX).")
+		cfg.AccessModes = append(cfg.AccessModes, v1.ReadWriteOnce, v1.ReadOnlyMany, v1.ReadOnlyMany)
+	}
+
+	return &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-",
+			Namespace:    ns,
+			Annotations:  cfg.Annotations,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			Selector:    cfg.Selector,
+			AccessModes: cfg.AccessModes,
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"),
+				},
+			},
+			StorageClassName: cfg.StorageClassName,
+			VolumeMode:       cfg.VolumeMode,
+		},
+	}
+}
+
+func createPDWithRetry(zone string) (string, error) {
+	var err error
+	for start := time.Now(); time.Since(start) < PDRetryTimeout; time.Sleep(PDRetryPollTime) {
+		newDiskName, err := createPD(zone)
+		if err != nil {
+			Logf("Couldn't create a new PD, sleeping 5 seconds: %v", err)
+			continue
+		}
+		Logf("Successfully created a new PD: %q.", newDiskName)
+		return newDiskName, nil
+	}
+	return "", err
+}
+
+func CreatePDWithRetry() (string, error) {
+	return createPDWithRetry("")
+}
+
+func CreatePDWithRetryAndZone(zone string) (string, error) {
+	return createPDWithRetry(zone)
+}
+
+func DeletePDWithRetry(diskName string) error {
+	var err error
+	for start := time.Now(); time.Since(start) < PDRetryTimeout; time.Sleep(PDRetryPollTime) {
+		err = deletePD(diskName)
+		if err != nil {
+			Logf("Couldn't delete PD %q, sleeping %v: %v", diskName, PDRetryPollTime, err)
+			continue
+		}
+		Logf("Successfully deleted PD %q.", diskName)
+		return nil
+	}
+	return fmt.Errorf("unable to delete PD %q: %v", diskName, err)
+}
+
+func createPD(zone string) (string, error) {
+	if zone == "" {
+		zone = TestContext.CloudConfig.Zone
+	}
+	return TestContext.CloudConfig.Provider.CreatePD(zone)
+}
+
+func deletePD(pdName string) error {
+	return TestContext.CloudConfig.Provider.DeletePD(pdName)
+}
+
+// Returns a pod definition based on the namespace. The pod references the PVC's
+// name.
+func MakeWritePod(ns string, pvc *v1.PersistentVolumeClaim) *v1.Pod {
+	return MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "touch /mnt/volume1/SUCCESS && (id -G | grep -E '\\b777\\b')")
+}
+
+// Returns a pod definition based on the namespace. The pod references the PVC's
+// name.  A slice of BASH commands can be supplied as args to be run by the pod
+func MakePod(ns string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *v1.Pod {
+	if len(command) == 0 {
+		command = "trap exit TERM; while true; do sleep 1; done"
+	}
+	podSpec := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-tester-",
+			Namespace:    ns,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    "write-pod",
+					Image:   BusyBoxImage,
+					Command: []string{"/bin/sh"},
+					Args:    []string{"-c", command},
+					SecurityContext: &v1.SecurityContext{
+						Privileged: &isPrivileged,
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyOnFailure,
+		},
+	}
+	var volumeMounts = make([]v1.VolumeMount, len(pvclaims))
+	var volumes = make([]v1.Volume, len(pvclaims))
+	for index, pvclaim := range pvclaims {
+		volumename := fmt.Sprintf("volume%v", index+1)
+		volumeMounts[index] = v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename}
+		volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}}
+	}
+	podSpec.Spec.Containers[0].VolumeMounts = volumeMounts
+	podSpec.Spec.Volumes = volumes
+	if nodeSelector != nil {
+		podSpec.Spec.NodeSelector = nodeSelector
+	}
+	return podSpec
+}
+
+// Returns a pod definition based on the namespace using nginx image
+func MakeNginxPod(ns string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim) *v1.Pod {
+	podSpec := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: "pvc-tester-",
+			Namespace:    ns,
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "write-pod",
+					Image: "nginx",
+					Ports: []v1.ContainerPort{
+						{
+							Name:          "http-server",
+							ContainerPort: 80,
+						},
+					},
+				},
+			},
+		},
+	}
+	var volumeMounts = make([]v1.VolumeMount, len(pvclaims))
+	var volumes = make([]v1.Volume, len(pvclaims))
+	for index, pvclaim := range pvclaims {
+		volumename := fmt.Sprintf("volume%v", index+1)
+		volumeMounts[index] = v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename}
+		volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}}
+	}
+	podSpec.Spec.Containers[0].VolumeMounts = volumeMounts
+	podSpec.Spec.Volumes = volumes
+	if nodeSelector != nil {
+		podSpec.Spec.NodeSelector = nodeSelector
+	}
+	return podSpec
+}
+
+// Returns a pod definition based on the namespace. The pod references the PVC's
+// name.  A slice of BASH commands can be supplied as args to be run by the pod.
+// SELinux testing requires to pass HostIPC and HostPID as booleansi arguments.
+func MakeSecPod(ns string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64) *v1.Pod {
+	if len(command) == 0 {
+		command = "trap exit TERM; while true; do sleep 1; done"
+	}
+	podName := "security-context-" + string(uuid.NewUUID())
+	if fsGroup == nil {
+		fsGroup = func(i int64) *int64 {
+			return &i
+		}(1000)
+	}
+	podSpec := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: ns,
+		},
+		Spec: v1.PodSpec{
+			HostIPC: hostIPC,
+			HostPID: hostPID,
+			SecurityContext: &v1.PodSecurityContext{
+				FSGroup: fsGroup,
+			},
+			Containers: []v1.Container{
+				{
+					Name:    "write-pod",
+					Image:   imageutils.GetE2EImage(imageutils.BusyBox),
+					Command: []string{"/bin/sh"},
+					Args:    []string{"-c", command},
+					SecurityContext: &v1.SecurityContext{
+						Privileged: &isPrivileged,
+					},
+				},
+			},
+			RestartPolicy: v1.RestartPolicyOnFailure,
+		},
+	}
+	var volumeMounts = make([]v1.VolumeMount, 0)
+	var volumeDevices = make([]v1.VolumeDevice, 0)
+	var volumes = make([]v1.Volume, len(pvclaims))
+	for index, pvclaim := range pvclaims {
+		volumename := fmt.Sprintf("volume%v", index+1)
+		if pvclaim.Spec.VolumeMode != nil && *pvclaim.Spec.VolumeMode == v1.PersistentVolumeBlock {
+			volumeDevices = append(volumeDevices, v1.VolumeDevice{Name: volumename, DevicePath: "/mnt/" + volumename})
+		} else {
+			volumeMounts = append(volumeMounts, v1.VolumeMount{Name: volumename, MountPath: "/mnt/" + volumename})
+		}
+
+		volumes[index] = v1.Volume{Name: volumename, VolumeSource: v1.VolumeSource{PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ClaimName: pvclaim.Name, ReadOnly: false}}}
+	}
+	podSpec.Spec.Containers[0].VolumeMounts = volumeMounts
+	podSpec.Spec.Containers[0].VolumeDevices = volumeDevices
+	podSpec.Spec.Volumes = volumes
+	podSpec.Spec.SecurityContext.SELinuxOptions = seLinuxLabel
+	return podSpec
+}
+
+// CreatePod with given claims based on node selector
+func CreatePod(client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) (*v1.Pod, error) {
+	pod := MakePod(namespace, nodeSelector, pvclaims, isPrivileged, command)
+	pod, err := client.CoreV1().Pods(namespace).Create(pod)
+	if err != nil {
+		return nil, fmt.Errorf("pod Create API error: %v", err)
+	}
+	// Waiting for pod to be running
+	err = WaitForPodNameRunningInNamespace(client, pod.Name, namespace)
+	if err != nil {
+		return pod, fmt.Errorf("pod %q is not Running: %v", pod.Name, err)
+	}
+	// get fresh pod info
+	pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
+	if err != nil {
+		return pod, fmt.Errorf("pod Get API error: %v", err)
+	}
+	return pod, nil
+}
+
+func CreateNginxPod(client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim) (*v1.Pod, error) {
+	pod := MakeNginxPod(namespace, nodeSelector, pvclaims)
+	pod, err := client.CoreV1().Pods(namespace).Create(pod)
+	if err != nil {
+		return nil, fmt.Errorf("pod Create API error: %v", err)
+	}
+	// Waiting for pod to be running
+	err = WaitForPodNameRunningInNamespace(client, pod.Name, namespace)
+	if err != nil {
+		return pod, fmt.Errorf("pod %q is not Running: %v", pod.Name, err)
+	}
+	// get fresh pod info
+	pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
+	if err != nil {
+		return pod, fmt.Errorf("pod Get API error: %v", err)
+	}
+	return pod, nil
+}
+
+// create security pod with given claims
+func CreateSecPod(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, timeout time.Duration) (*v1.Pod, error) {
+	return CreateSecPodWithNodeName(client, namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup, "", timeout)
+}
+
+// create security pod with given claims
+func CreateSecPodWithNodeName(client clientset.Interface, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string, hostIPC bool, hostPID bool, seLinuxLabel *v1.SELinuxOptions, fsGroup *int64, nodeName string, timeout time.Duration) (*v1.Pod, error) {
+	pod := MakeSecPod(namespace, pvclaims, isPrivileged, command, hostIPC, hostPID, seLinuxLabel, fsGroup)
+	// Setting nodeName
+	pod.Spec.NodeName = nodeName
+
+	pod, err := client.CoreV1().Pods(namespace).Create(pod)
+	if err != nil {
+		return nil, fmt.Errorf("pod Create API error: %v", err)
+	}
+
+	// Waiting for pod to be running
+	err = WaitTimeoutForPodRunningInNamespace(client, pod.Name, namespace, timeout)
+	if err != nil {
+		return pod, fmt.Errorf("pod %q is not Running: %v", pod.Name, err)
+	}
+	// get fresh pod info
+	pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
+	if err != nil {
+		return pod, fmt.Errorf("pod Get API error: %v", err)
+	}
+	return pod, nil
+}
+
+// Define and create a pod with a mounted PV.  Pod runs infinite loop until killed.
+func CreateClientPod(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.Pod, error) {
+	return CreatePod(c, ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "")
+}
+
+// CreateUnschedulablePod with given claims based on node selector
+func CreateUnschedulablePod(client clientset.Interface, namespace string, nodeSelector map[string]string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) (*v1.Pod, error) {
+	pod := MakePod(namespace, nodeSelector, pvclaims, isPrivileged, command)
+	pod, err := client.CoreV1().Pods(namespace).Create(pod)
+	if err != nil {
+		return nil, fmt.Errorf("pod Create API error: %v", err)
+	}
+	// Waiting for pod to become Unschedulable
+	err = WaitForPodNameUnschedulableInNamespace(client, pod.Name, namespace)
+	if err != nil {
+		return pod, fmt.Errorf("pod %q is not Unschedulable: %v", pod.Name, err)
+	}
+	// get fresh pod info
+	pod, err = client.CoreV1().Pods(namespace).Get(pod.Name, metav1.GetOptions{})
+	if err != nil {
+		return pod, fmt.Errorf("pod Get API error: %v", err)
+	}
+	return pod, nil
+}
+
+// wait until all pvcs phase set to bound
+func WaitForPVClaimBoundPhase(client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, timeout time.Duration) ([]*v1.PersistentVolume, error) {
+	persistentvolumes := make([]*v1.PersistentVolume, len(pvclaims))
+
+	for index, claim := range pvclaims {
+		err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, Poll, timeout)
+		if err != nil {
+			return persistentvolumes, err
+		}
+		// Get new copy of the claim
+		claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+		if err != nil {
+			return persistentvolumes, fmt.Errorf("PVC Get API error: %v", err)
+		}
+		// Get the bounded PV
+		persistentvolumes[index], err = client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+		if err != nil {
+			return persistentvolumes, fmt.Errorf("PV Get API error: %v", err)
+		}
+	}
+	return persistentvolumes, nil
+}
+
+func CreatePVSource(zone string) (*v1.PersistentVolumeSource, error) {
+	diskName, err := CreatePDWithRetryAndZone(zone)
+	if err != nil {
+		return nil, err
+	}
+	return TestContext.CloudConfig.Provider.CreatePVSource(zone, diskName)
+}
+
+func DeletePVSource(pvSource *v1.PersistentVolumeSource) error {
+	return TestContext.CloudConfig.Provider.DeletePVSource(pvSource)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/rc_util.go

@@ -0,0 +1,293 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	scaleclient "k8s.io/client-go/scale"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	testutils "k8s.io/kubernetes/test/utils"
+)
+
+// RcByNamePort returns a ReplicationController with specified name and port
+func RcByNamePort(name string, replicas int32, image string, port int, protocol v1.Protocol,
+	labels map[string]string, gracePeriod *int64) *v1.ReplicationController {
+
+	return RcByNameContainer(name, replicas, image, labels, v1.Container{
+		Name:  name,
+		Image: image,
+		Ports: []v1.ContainerPort{{ContainerPort: int32(port), Protocol: protocol}},
+	}, gracePeriod)
+}
+
+// RcByNameContainer returns a ReplicationController with specified name and container
+func RcByNameContainer(name string, replicas int32, image string, labels map[string]string, c v1.Container,
+	gracePeriod *int64) *v1.ReplicationController {
+
+	zeroGracePeriod := int64(0)
+
+	// Add "name": name to the labels, overwriting if it exists.
+	labels["name"] = name
+	if gracePeriod == nil {
+		gracePeriod = &zeroGracePeriod
+	}
+	return &v1.ReplicationController{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ReplicationController",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.ReplicationControllerSpec{
+			Replicas: func(i int32) *int32 { return &i }(replicas),
+			Selector: map[string]string{
+				"name": name,
+			},
+			Template: &v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: labels,
+				},
+				Spec: v1.PodSpec{
+					Containers:                    []v1.Container{c},
+					TerminationGracePeriodSeconds: gracePeriod,
+				},
+			},
+		},
+	}
+}
+
+// ScaleRCByLabels scales an RC via ns/label lookup. If replicas == 0 it waits till
+// none are running, otherwise it does what a synchronous scale operation would do.
+func ScaleRCByLabels(clientset clientset.Interface, scalesGetter scaleclient.ScalesGetter, ns string, l map[string]string, replicas uint) error {
+	listOpts := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(l)).String()}
+	rcs, err := clientset.CoreV1().ReplicationControllers(ns).List(listOpts)
+	if err != nil {
+		return err
+	}
+	if len(rcs.Items) == 0 {
+		return fmt.Errorf("RC with labels %v not found in ns %v", l, ns)
+	}
+	Logf("Scaling %v RCs with labels %v in ns %v to %v replicas.", len(rcs.Items), l, ns, replicas)
+	for _, labelRC := range rcs.Items {
+		name := labelRC.Name
+		if err := ScaleRC(clientset, scalesGetter, ns, name, replicas, false); err != nil {
+			return err
+		}
+		rc, err := clientset.CoreV1().ReplicationControllers(ns).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if replicas == 0 {
+			ps, err := testutils.NewPodStore(clientset, rc.Namespace, labels.SelectorFromSet(rc.Spec.Selector), fields.Everything())
+			if err != nil {
+				return err
+			}
+			defer ps.Stop()
+			if err = waitForPodsGone(ps, 10*time.Second, 10*time.Minute); err != nil {
+				return fmt.Errorf("error while waiting for pods gone %s: %v", name, err)
+			}
+		} else {
+			if err := testutils.WaitForPodsWithLabelRunning(
+				clientset, ns, labels.SelectorFromSet(labels.Set(rc.Spec.Selector))); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+type updateRcFunc func(d *v1.ReplicationController)
+
+func UpdateReplicationControllerWithRetries(c clientset.Interface, namespace, name string, applyUpdate updateRcFunc) (*v1.ReplicationController, error) {
+	var rc *v1.ReplicationController
+	var updateErr error
+	pollErr := wait.PollImmediate(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
+		var err error
+		if rc, err = c.CoreV1().ReplicationControllers(namespace).Get(name, metav1.GetOptions{}); err != nil {
+			return false, err
+		}
+		// Apply the update, then attempt to push it to the apiserver.
+		applyUpdate(rc)
+		if rc, err = c.CoreV1().ReplicationControllers(namespace).Update(rc); err == nil {
+			Logf("Updating replication controller %q", name)
+			return true, nil
+		}
+		updateErr = err
+		return false, nil
+	})
+	if pollErr == wait.ErrWaitTimeout {
+		pollErr = fmt.Errorf("couldn't apply the provided updated to rc %q: %v", name, updateErr)
+	}
+	return rc, pollErr
+}
+
+// DeleteRCAndWaitForGC deletes only the Replication Controller and waits for GC to delete the pods.
+func DeleteRCAndWaitForGC(c clientset.Interface, ns, name string) error {
+	return DeleteResourceAndWaitForGC(c, api.Kind("ReplicationController"), ns, name)
+}
+
+func ScaleRC(clientset clientset.Interface, scalesGetter scaleclient.ScalesGetter, ns, name string, size uint, wait bool) error {
+	return ScaleResource(clientset, scalesGetter, ns, name, size, wait, api.Kind("ReplicationController"), api.Resource("replicationcontrollers"))
+}
+
+func RunRC(config testutils.RCConfig) error {
+	By(fmt.Sprintf("creating replication controller %s in namespace %s", config.Name, config.Namespace))
+	config.NodeDumpFunc = DumpNodeDebugInfo
+	config.ContainerDumpFunc = LogFailedContainers
+	return testutils.RunRC(config)
+}
+
+// WaitForRCPodToDisappear returns nil if the pod from the given replication controller (described by rcName) no longer exists.
+// In case of failure or too long waiting time, an error is returned.
+func WaitForRCPodToDisappear(c clientset.Interface, ns, rcName, podName string) error {
+	label := labels.SelectorFromSet(labels.Set(map[string]string{"name": rcName}))
+	// NodeController evicts pod after 5 minutes, so we need timeout greater than that to observe effects.
+	// The grace period must be set to 0 on the pod for it to be deleted during the partition.
+	// Otherwise, it goes to the 'Terminating' state till the kubelet confirms deletion.
+	return WaitForPodToDisappear(c, ns, podName, label, 20*time.Second, 10*time.Minute)
+}
+
+// WaitForReplicationController waits until the RC appears (exist == true), or disappears (exist == false)
+func WaitForReplicationController(c clientset.Interface, namespace, name string, exist bool, interval, timeout time.Duration) error {
+	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
+		_, err := c.CoreV1().ReplicationControllers(namespace).Get(name, metav1.GetOptions{})
+		if err != nil {
+			Logf("Get ReplicationController %s in namespace %s failed (%v).", name, namespace, err)
+			return !exist, nil
+		} else {
+			Logf("ReplicationController %s in namespace %s found.", name, namespace)
+			return exist, nil
+		}
+	})
+	if err != nil {
+		stateMsg := map[bool]string{true: "to appear", false: "to disappear"}
+		return fmt.Errorf("error waiting for ReplicationController %s/%s %s: %v", namespace, name, stateMsg[exist], err)
+	}
+	return nil
+}
+
+// WaitForReplicationControllerwithSelector waits until any RC with given selector appears (exist == true), or disappears (exist == false)
+func WaitForReplicationControllerwithSelector(c clientset.Interface, namespace string, selector labels.Selector, exist bool, interval,
+	timeout time.Duration) error {
+	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
+		rcs, err := c.CoreV1().ReplicationControllers(namespace).List(metav1.ListOptions{LabelSelector: selector.String()})
+		switch {
+		case len(rcs.Items) != 0:
+			Logf("ReplicationController with %s in namespace %s found.", selector.String(), namespace)
+			return exist, nil
+		case len(rcs.Items) == 0:
+			Logf("ReplicationController with %s in namespace %s disappeared.", selector.String(), namespace)
+			return !exist, nil
+		default:
+			Logf("List ReplicationController with %s in namespace %s failed: %v", selector.String(), namespace, err)
+			return false, nil
+		}
+	})
+	if err != nil {
+		stateMsg := map[bool]string{true: "to appear", false: "to disappear"}
+		return fmt.Errorf("error waiting for ReplicationControllers with %s in namespace %s %s: %v", selector.String(), namespace, stateMsg[exist], err)
+	}
+	return nil
+}
+
+// trimDockerRegistry is the function for trimming the docker.io/library from the beginning of the imagename.
+// If community docker installed it will not prefix the registry names with the dockerimages vs registry names prefixed with other runtimes or docker installed via RHEL extra repo.
+// So this function will help to trim the docker.io/library if exists
+func trimDockerRegistry(imagename string) string {
+	imagename = strings.Replace(imagename, "docker.io/", "", 1)
+	return strings.Replace(imagename, "library/", "", 1)
+}
+
+// validatorFn is the function which is individual tests will implement.
+// we may want it to return more than just an error, at some point.
+type validatorFn func(c clientset.Interface, podID string) error
+
+// ValidateController is a generic mechanism for testing RC's that are running.
+// It takes a container name, a test name, and a validator function which is plugged in by a specific test.
+// "containername": this is grepped for.
+// "containerImage" : this is the name of the image we expect to be launched.  Not to confuse w/ images (kitten.jpg)  which are validated.
+// "testname":  which gets bubbled up to the logging/failure messages if errors happen.
+// "validator" function: This function is given a podID and a client, and it can do some specific validations that way.
+func ValidateController(c clientset.Interface, containerImage string, replicas int, containername string, testname string, validator validatorFn, ns string) {
+	containerImage = trimDockerRegistry(containerImage)
+	getPodsTemplate := "--template={{range.items}}{{.metadata.name}} {{end}}"
+	// NB: kubectl adds the "exists" function to the standard template functions.
+	// This lets us check to see if the "running" entry exists for each of the containers
+	// we care about. Exists will never return an error and it's safe to check a chain of
+	// things, any one of which may not exist. In the below template, all of info,
+	// containername, and running might be nil, so the normal index function isn't very
+	// helpful.
+	// This template is unit-tested in kubectl, so if you change it, update the unit test.
+	// You can read about the syntax here: http://golang.org/pkg/text/template/.
+	getContainerStateTemplate := fmt.Sprintf(`--template={{if (exists . "status" "containerStatuses")}}{{range .status.containerStatuses}}{{if (and (eq .name "%s") (exists . "state" "running"))}}true{{end}}{{end}}{{end}}`, containername)
+
+	getImageTemplate := fmt.Sprintf(`--template={{if (exists . "spec" "containers")}}{{range .spec.containers}}{{if eq .name "%s"}}{{.image}}{{end}}{{end}}{{end}}`, containername)
+
+	By(fmt.Sprintf("waiting for all containers in %s pods to come up.", testname)) //testname should be selector
+waitLoop:
+	for start := time.Now(); time.Since(start) < PodStartTimeout; time.Sleep(5 * time.Second) {
+		getPodsOutput := RunKubectlOrDie("get", "pods", "-o", "template", getPodsTemplate, "-l", testname, fmt.Sprintf("--namespace=%v", ns))
+		pods := strings.Fields(getPodsOutput)
+		if numPods := len(pods); numPods != replicas {
+			By(fmt.Sprintf("Replicas for %s: expected=%d actual=%d", testname, replicas, numPods))
+			continue
+		}
+		var runningPods []string
+		for _, podID := range pods {
+			running := RunKubectlOrDie("get", "pods", podID, "-o", "template", getContainerStateTemplate, fmt.Sprintf("--namespace=%v", ns))
+			if running != "true" {
+				Logf("%s is created but not running", podID)
+				continue waitLoop
+			}
+
+			currentImage := RunKubectlOrDie("get", "pods", podID, "-o", "template", getImageTemplate, fmt.Sprintf("--namespace=%v", ns))
+			currentImage = trimDockerRegistry(currentImage)
+			if currentImage != containerImage {
+				Logf("%s is created but running wrong image; expected: %s, actual: %s", podID, containerImage, currentImage)
+				continue waitLoop
+			}
+
+			// Call the generic validator function here.
+			// This might validate for example, that (1) getting a url works and (2) url is serving correct content.
+			if err := validator(c, podID); err != nil {
+				Logf("%s is running right image but validator function failed: %v", podID, err)
+				continue waitLoop
+			}
+
+			Logf("%s is verified up and running", podID)
+			runningPods = append(runningPods, podID)
+		}
+		// If we reach here, then all our checks passed.
+		if len(runningPods) == replicas {
+			return
+		}
+	}
+	// Reaching here means that one of more checks failed multiple times.  Assuming its not a race condition, something is broken.
+	Failf("Timed out after %v seconds waiting for %s pods to reach valid state", PodStartTimeout.Seconds(), testname)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/resource_usage_gatherer.go

@@ -0,0 +1,399 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/util/system"
+)
+
+type ResourceConstraint struct {
+	CPUConstraint    float64
+	MemoryConstraint uint64
+}
+
+type SingleContainerSummary struct {
+	Name string
+	Cpu  float64
+	Mem  uint64
+}
+
+// we can't have int here, as JSON does not accept integer keys.
+type ResourceUsageSummary map[string][]SingleContainerSummary
+
+const NoCPUConstraint = math.MaxFloat64
+
+func (s *ResourceUsageSummary) PrintHumanReadable() string {
+	buf := &bytes.Buffer{}
+	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+	for perc, summaries := range *s {
+		buf.WriteString(fmt.Sprintf("%v percentile:\n", perc))
+		fmt.Fprintf(w, "container\tcpu(cores)\tmemory(MB)\n")
+		for _, summary := range summaries {
+			fmt.Fprintf(w, "%q\t%.3f\t%.2f\n", summary.Name, summary.Cpu, float64(summary.Mem)/(1024*1024))
+		}
+		w.Flush()
+	}
+	return buf.String()
+}
+
+func (s *ResourceUsageSummary) PrintJSON() string {
+	return PrettyPrintJSON(*s)
+}
+
+func (s *ResourceUsageSummary) SummaryKind() string {
+	return "ResourceUsageSummary"
+}
+
+func computePercentiles(timeSeries []ResourceUsagePerContainer, percentilesToCompute []int) map[int]ResourceUsagePerContainer {
+	if len(timeSeries) == 0 {
+		return make(map[int]ResourceUsagePerContainer)
+	}
+	dataMap := make(map[string]*usageDataPerContainer)
+	for i := range timeSeries {
+		for name, data := range timeSeries[i] {
+			if dataMap[name] == nil {
+				dataMap[name] = &usageDataPerContainer{
+					cpuData:        make([]float64, 0, len(timeSeries)),
+					memUseData:     make([]uint64, 0, len(timeSeries)),
+					memWorkSetData: make([]uint64, 0, len(timeSeries)),
+				}
+			}
+			dataMap[name].cpuData = append(dataMap[name].cpuData, data.CPUUsageInCores)
+			dataMap[name].memUseData = append(dataMap[name].memUseData, data.MemoryUsageInBytes)
+			dataMap[name].memWorkSetData = append(dataMap[name].memWorkSetData, data.MemoryWorkingSetInBytes)
+		}
+	}
+	for _, v := range dataMap {
+		sort.Float64s(v.cpuData)
+		sort.Sort(uint64arr(v.memUseData))
+		sort.Sort(uint64arr(v.memWorkSetData))
+	}
+
+	result := make(map[int]ResourceUsagePerContainer)
+	for _, perc := range percentilesToCompute {
+		data := make(ResourceUsagePerContainer)
+		for k, v := range dataMap {
+			percentileIndex := int(math.Ceil(float64(len(v.cpuData)*perc)/100)) - 1
+			data[k] = &ContainerResourceUsage{
+				Name:                    k,
+				CPUUsageInCores:         v.cpuData[percentileIndex],
+				MemoryUsageInBytes:      v.memUseData[percentileIndex],
+				MemoryWorkingSetInBytes: v.memWorkSetData[percentileIndex],
+			}
+		}
+		result[perc] = data
+	}
+	return result
+}
+
+func leftMergeData(left, right map[int]ResourceUsagePerContainer) map[int]ResourceUsagePerContainer {
+	result := make(map[int]ResourceUsagePerContainer)
+	for percentile, data := range left {
+		result[percentile] = data
+		if _, ok := right[percentile]; !ok {
+			continue
+		}
+		for k, v := range right[percentile] {
+			result[percentile][k] = v
+		}
+	}
+	return result
+}
+
+type resourceGatherWorker struct {
+	c                           clientset.Interface
+	nodeName                    string
+	wg                          *sync.WaitGroup
+	containerIDs                []string
+	stopCh                      chan struct{}
+	dataSeries                  []ResourceUsagePerContainer
+	finished                    bool
+	inKubemark                  bool
+	resourceDataGatheringPeriod time.Duration
+	probeDuration               time.Duration
+	printVerboseLogs            bool
+}
+
+func (w *resourceGatherWorker) singleProbe() {
+	data := make(ResourceUsagePerContainer)
+	if w.inKubemark {
+		kubemarkData := GetKubemarkMasterComponentsResourceUsage()
+		if data == nil {
+			return
+		}
+		for k, v := range kubemarkData {
+			data[k] = &ContainerResourceUsage{
+				Name:                    v.Name,
+				MemoryWorkingSetInBytes: v.MemoryWorkingSetInBytes,
+				CPUUsageInCores:         v.CPUUsageInCores,
+			}
+		}
+	} else {
+		nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, w.probeDuration, func() []string { return w.containerIDs })
+		if err != nil {
+			Logf("Error while reading data from %v: %v", w.nodeName, err)
+			return
+		}
+		for k, v := range nodeUsage {
+			data[k] = v
+			if w.printVerboseLogs {
+				Logf("Get container %v usage on node %v. CPUUsageInCores: %v, MemoryUsageInBytes: %v, MemoryWorkingSetInBytes: %v", k, w.nodeName, v.CPUUsageInCores, v.MemoryUsageInBytes, v.MemoryWorkingSetInBytes)
+			}
+		}
+	}
+	w.dataSeries = append(w.dataSeries, data)
+}
+
+func (w *resourceGatherWorker) gather(initialSleep time.Duration) {
+	defer utilruntime.HandleCrash()
+	defer w.wg.Done()
+	defer Logf("Closing worker for %v", w.nodeName)
+	defer func() { w.finished = true }()
+	select {
+	case <-time.After(initialSleep):
+		w.singleProbe()
+		for {
+			select {
+			case <-time.After(w.resourceDataGatheringPeriod):
+				w.singleProbe()
+			case <-w.stopCh:
+				return
+			}
+		}
+	case <-w.stopCh:
+		return
+	}
+}
+
+type ContainerResourceGatherer struct {
+	client       clientset.Interface
+	stopCh       chan struct{}
+	workers      []resourceGatherWorker
+	workerWg     sync.WaitGroup
+	containerIDs []string
+	options      ResourceGathererOptions
+}
+
+type ResourceGathererOptions struct {
+	InKubemark                  bool
+	Nodes                       NodesSet
+	ResourceDataGatheringPeriod time.Duration
+	ProbeDuration               time.Duration
+	PrintVerboseLogs            bool
+}
+
+type NodesSet int
+
+const (
+	AllNodes          NodesSet = 0 // All containers on all nodes
+	MasterNodes       NodesSet = 1 // All containers on Master nodes only
+	MasterAndDNSNodes NodesSet = 2 // All containers on Master nodes and DNS containers on other nodes
+)
+
+func NewResourceUsageGatherer(c clientset.Interface, options ResourceGathererOptions, pods *v1.PodList) (*ContainerResourceGatherer, error) {
+	g := ContainerResourceGatherer{
+		client:       c,
+		stopCh:       make(chan struct{}),
+		containerIDs: make([]string, 0),
+		options:      options,
+	}
+
+	if options.InKubemark {
+		g.workerWg.Add(1)
+		g.workers = append(g.workers, resourceGatherWorker{
+			inKubemark:                  true,
+			stopCh:                      g.stopCh,
+			wg:                          &g.workerWg,
+			finished:                    false,
+			resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod,
+			probeDuration:               options.ProbeDuration,
+			printVerboseLogs:            options.PrintVerboseLogs,
+		})
+	} else {
+		// Tracks kube-system pods if no valid PodList is passed in.
+		var err error
+		if pods == nil {
+			pods, err = c.CoreV1().Pods("kube-system").List(metav1.ListOptions{})
+			if err != nil {
+				Logf("Error while listing Pods: %v", err)
+				return nil, err
+			}
+		}
+		dnsNodes := make(map[string]bool)
+		for _, pod := range pods.Items {
+			if (options.Nodes == MasterNodes) && !system.IsMasterNode(pod.Spec.NodeName) {
+				continue
+			}
+			if (options.Nodes == MasterAndDNSNodes) && !system.IsMasterNode(pod.Spec.NodeName) && pod.Labels["k8s-app"] != "kube-dns" {
+				continue
+			}
+			for _, container := range pod.Status.InitContainerStatuses {
+				g.containerIDs = append(g.containerIDs, container.Name)
+			}
+			for _, container := range pod.Status.ContainerStatuses {
+				g.containerIDs = append(g.containerIDs, container.Name)
+			}
+			if options.Nodes == MasterAndDNSNodes {
+				dnsNodes[pod.Spec.NodeName] = true
+			}
+		}
+		nodeList, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
+		if err != nil {
+			Logf("Error while listing Nodes: %v", err)
+			return nil, err
+		}
+
+		for _, node := range nodeList.Items {
+			if options.Nodes == AllNodes || system.IsMasterNode(node.Name) || dnsNodes[node.Name] {
+				g.workerWg.Add(1)
+				g.workers = append(g.workers, resourceGatherWorker{
+					c:                           c,
+					nodeName:                    node.Name,
+					wg:                          &g.workerWg,
+					containerIDs:                g.containerIDs,
+					stopCh:                      g.stopCh,
+					finished:                    false,
+					inKubemark:                  false,
+					resourceDataGatheringPeriod: options.ResourceDataGatheringPeriod,
+					probeDuration:               options.ProbeDuration,
+					printVerboseLogs:            options.PrintVerboseLogs,
+				})
+				if options.Nodes == MasterNodes {
+					break
+				}
+			}
+		}
+	}
+	return &g, nil
+}
+
+// StartGatheringData starts a stat gathering worker blocks for each node to track,
+// and blocks until StopAndSummarize is called.
+func (g *ContainerResourceGatherer) StartGatheringData() {
+	if len(g.workers) == 0 {
+		return
+	}
+	delayPeriod := g.options.ResourceDataGatheringPeriod / time.Duration(len(g.workers))
+	delay := time.Duration(0)
+	for i := range g.workers {
+		go g.workers[i].gather(delay)
+		delay += delayPeriod
+	}
+	g.workerWg.Wait()
+}
+
+// StopAndSummarize stops stat gathering workers, processes the collected stats,
+// generates resource summary for the passed-in percentiles, and returns the summary.
+// It returns an error if the resource usage at any percentile is beyond the
+// specified resource constraints.
+func (g *ContainerResourceGatherer) StopAndSummarize(percentiles []int, constraints map[string]ResourceConstraint) (*ResourceUsageSummary, error) {
+	close(g.stopCh)
+	Logf("Closed stop channel. Waiting for %v workers", len(g.workers))
+	finished := make(chan struct{})
+	go func() {
+		g.workerWg.Wait()
+		finished <- struct{}{}
+	}()
+	select {
+	case <-finished:
+		Logf("Waitgroup finished.")
+	case <-time.After(2 * time.Minute):
+		unfinished := make([]string, 0)
+		for i := range g.workers {
+			if !g.workers[i].finished {
+				unfinished = append(unfinished, g.workers[i].nodeName)
+			}
+		}
+		Logf("Timed out while waiting for waitgroup, some workers failed to finish: %v", unfinished)
+	}
+
+	if len(percentiles) == 0 {
+		Logf("Warning! Empty percentile list for stopAndPrintData.")
+		return &ResourceUsageSummary{}, fmt.Errorf("Failed to get any resource usage data")
+	}
+	data := make(map[int]ResourceUsagePerContainer)
+	for i := range g.workers {
+		if g.workers[i].finished {
+			stats := computePercentiles(g.workers[i].dataSeries, percentiles)
+			data = leftMergeData(stats, data)
+		}
+	}
+
+	// Workers has been stopped. We need to gather data stored in them.
+	sortedKeys := []string{}
+	for name := range data[percentiles[0]] {
+		sortedKeys = append(sortedKeys, name)
+	}
+	sort.Strings(sortedKeys)
+	violatedConstraints := make([]string, 0)
+	summary := make(ResourceUsageSummary)
+	for _, perc := range percentiles {
+		for _, name := range sortedKeys {
+			usage := data[perc][name]
+			summary[strconv.Itoa(perc)] = append(summary[strconv.Itoa(perc)], SingleContainerSummary{
+				Name: name,
+				Cpu:  usage.CPUUsageInCores,
+				Mem:  usage.MemoryWorkingSetInBytes,
+			})
+			// Verifying 99th percentile of resource usage
+			if perc == 99 {
+				// Name has a form: <pod_name>/<container_name>
+				containerName := strings.Split(name, "/")[1]
+				if constraint, ok := constraints[containerName]; ok {
+					if usage.CPUUsageInCores > constraint.CPUConstraint {
+						violatedConstraints = append(
+							violatedConstraints,
+							fmt.Sprintf("Container %v is using %v/%v CPU",
+								name,
+								usage.CPUUsageInCores,
+								constraint.CPUConstraint,
+							),
+						)
+					}
+					if usage.MemoryWorkingSetInBytes > constraint.MemoryConstraint {
+						violatedConstraints = append(
+							violatedConstraints,
+							fmt.Sprintf("Container %v is using %v/%v MB of memory",
+								name,
+								float64(usage.MemoryWorkingSetInBytes)/(1024*1024),
+								float64(constraint.MemoryConstraint)/(1024*1024),
+							),
+						)
+					}
+				}
+			}
+		}
+	}
+	if len(violatedConstraints) > 0 {
+		return &summary, fmt.Errorf(strings.Join(violatedConstraints, "\n"))
+	}
+	return &summary, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/rs_util.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo"
+
+	apps "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	appsclient "k8s.io/client-go/kubernetes/typed/apps/v1"
+	deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util"
+	testutils "k8s.io/kubernetes/test/utils"
+)
+
+type updateRsFunc func(d *apps.ReplicaSet)
+
+func UpdateReplicaSetWithRetries(c clientset.Interface, namespace, name string, applyUpdate testutils.UpdateReplicaSetFunc) (*apps.ReplicaSet, error) {
+	return testutils.UpdateReplicaSetWithRetries(c, namespace, name, applyUpdate, Logf, Poll, pollShortTimeout)
+}
+
+// CheckNewRSAnnotations check if the new RS's annotation is as expected
+func CheckNewRSAnnotations(c clientset.Interface, ns, deploymentName string, expectedAnnotations map[string]string) error {
+	deployment, err := c.AppsV1().Deployments(ns).Get(deploymentName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	newRS, err := deploymentutil.GetNewReplicaSet(deployment, c.AppsV1())
+	if err != nil {
+		return err
+	}
+	for k, v := range expectedAnnotations {
+		// Skip checking revision annotations
+		if k != deploymentutil.RevisionAnnotation && v != newRS.Annotations[k] {
+			return fmt.Errorf("Expected new RS annotations = %+v, got %+v", expectedAnnotations, newRS.Annotations)
+		}
+	}
+	return nil
+}
+
+// WaitForReadyReplicaSet waits until the replicaset has all of its replicas ready.
+func WaitForReadyReplicaSet(c clientset.Interface, ns, name string) error {
+	err := wait.Poll(Poll, pollShortTimeout, func() (bool, error) {
+		rs, err := c.AppsV1().ReplicaSets(ns).Get(name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return *(rs.Spec.Replicas) == rs.Status.Replicas && *(rs.Spec.Replicas) == rs.Status.ReadyReplicas, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("replicaset %q never became ready", name)
+	}
+	return err
+}
+
+// WaitForReplicaSetDesiredReplicas waits until the replicaset has desired number of replicas.
+func WaitForReplicaSetDesiredReplicas(rsClient appsclient.ReplicaSetsGetter, replicaSet *apps.ReplicaSet) error {
+	desiredGeneration := replicaSet.Generation
+	err := wait.PollImmediate(Poll, pollShortTimeout, func() (bool, error) {
+		rs, err := rsClient.ReplicaSets(replicaSet.Namespace).Get(replicaSet.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return rs.Status.ObservedGeneration >= desiredGeneration && rs.Status.Replicas == *(replicaSet.Spec.Replicas) && rs.Status.Replicas == *(rs.Spec.Replicas), nil
+	})
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("replicaset %q never had desired number of replicas", replicaSet.Name)
+	}
+	return err
+}
+
+// WaitForReplicaSetTargetSpecReplicas waits for .spec.replicas of a RS to equal targetReplicaNum
+func WaitForReplicaSetTargetSpecReplicas(c clientset.Interface, replicaSet *apps.ReplicaSet, targetReplicaNum int32) error {
+	desiredGeneration := replicaSet.Generation
+	err := wait.PollImmediate(Poll, pollShortTimeout, func() (bool, error) {
+		rs, err := c.AppsV1().ReplicaSets(replicaSet.Namespace).Get(replicaSet.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return rs.Status.ObservedGeneration >= desiredGeneration && *rs.Spec.Replicas == targetReplicaNum, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("replicaset %q never had desired number of .spec.replicas", replicaSet.Name)
+	}
+	return err
+}
+
+// WaitForReplicaSetTargetAvailableReplicas waits for .status.availableReplicas of a RS to equal targetReplicaNum
+func WaitForReplicaSetTargetAvailableReplicas(c clientset.Interface, replicaSet *apps.ReplicaSet, targetReplicaNum int32) error {
+	desiredGeneration := replicaSet.Generation
+	err := wait.PollImmediate(Poll, pollShortTimeout, func() (bool, error) {
+		rs, err := c.AppsV1().ReplicaSets(replicaSet.Namespace).Get(replicaSet.Name, metav1.GetOptions{})
+		if err != nil {
+			return false, err
+		}
+		return rs.Status.ObservedGeneration >= desiredGeneration && rs.Status.AvailableReplicas == targetReplicaNum, nil
+	})
+	if err == wait.ErrWaitTimeout {
+		err = fmt.Errorf("replicaset %q never had desired number of .status.availableReplicas", replicaSet.Name)
+	}
+	return err
+}
+
+func RunReplicaSet(config testutils.ReplicaSetConfig) error {
+	By(fmt.Sprintf("creating replicaset %s in namespace %s", config.Name, config.Namespace))
+	config.NodeDumpFunc = DumpNodeDebugInfo
+	config.ContainerDumpFunc = LogFailedContainers
+	return testutils.RunReplicaSet(config)
+}
+
+func NewReplicaSet(name, namespace string, replicas int32, podLabels map[string]string, imageName, image string) *apps.ReplicaSet {
+	return &apps.ReplicaSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ReplicaSet",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+		Spec: apps.ReplicaSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: podLabels,
+			},
+			Replicas: &replicas,
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: podLabels,
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  imageName,
+							Image: image,
+						},
+					},
+				},
+			},
+		},
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/size.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	resizeNodeReadyTimeout    = 2 * time.Minute
+	resizeNodeNotReadyTimeout = 2 * time.Minute
+)
+
+func ResizeGroup(group string, size int32) error {
+	if TestContext.ReportDir != "" {
+		CoreDump(TestContext.ReportDir)
+		defer CoreDump(TestContext.ReportDir)
+	}
+	return TestContext.CloudConfig.Provider.ResizeGroup(group, size)
+}
+
+func GetGroupNodes(group string) ([]string, error) {
+	return TestContext.CloudConfig.Provider.GetGroupNodes(group)
+}
+
+func GroupSize(group string) (int, error) {
+	return TestContext.CloudConfig.Provider.GroupSize(group)
+}
+
+func WaitForGroupSize(group string, size int32) error {
+	timeout := 30 * time.Minute
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(20 * time.Second) {
+		currentSize, err := GroupSize(group)
+		if err != nil {
+			Logf("Failed to get node instance group size: %v", err)
+			continue
+		}
+		if currentSize != int(size) {
+			Logf("Waiting for node instance group size %d, current size %d", size, currentSize)
+			continue
+		}
+		Logf("Node instance group has reached the desired size %d", size)
+		return nil
+	}
+	return fmt.Errorf("timeout waiting %v for node instance group size to be %d", timeout, size)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/ssh.go

@@ -0,0 +1,283 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	sshutil "k8s.io/kubernetes/pkg/ssh"
+)
+
+// GetSigner returns an ssh.Signer for the provider ("gce", etc.) that can be
+// used to SSH to their nodes.
+func GetSigner(provider string) (ssh.Signer, error) {
+	// honor a consistent SSH key across all providers
+	if path := os.Getenv("KUBE_SSH_KEY_PATH"); len(path) > 0 {
+		return sshutil.MakePrivateKeySignerFromFile(path)
+	}
+
+	// Select the key itself to use. When implementing more providers here,
+	// please also add them to any SSH tests that are disabled because of signer
+	// support.
+	keyfile := ""
+	switch provider {
+	case "gce", "gke", "kubemark":
+		keyfile = os.Getenv("GCE_SSH_KEY")
+		if keyfile == "" {
+			keyfile = "google_compute_engine"
+		}
+	case "aws", "eks":
+		keyfile = os.Getenv("AWS_SSH_KEY")
+		if keyfile == "" {
+			keyfile = "kube_aws_rsa"
+		}
+	case "local", "vsphere":
+		keyfile = os.Getenv("LOCAL_SSH_KEY")
+		if keyfile == "" {
+			keyfile = "id_rsa"
+		}
+	case "skeleton":
+		keyfile = os.Getenv("KUBE_SSH_KEY")
+		if keyfile == "" {
+			keyfile = "id_rsa"
+		}
+	default:
+		return nil, fmt.Errorf("GetSigner(...) not implemented for %s", provider)
+	}
+
+	// Respect absolute paths for keys given by user, fallback to assuming
+	// relative paths are in ~/.ssh
+	if !filepath.IsAbs(keyfile) {
+		keydir := filepath.Join(os.Getenv("HOME"), ".ssh")
+		keyfile = filepath.Join(keydir, keyfile)
+	}
+
+	return sshutil.MakePrivateKeySignerFromFile(keyfile)
+}
+
+// NodeSSHHosts returns SSH-able host names for all schedulable nodes - this
+// excludes master node. If it can't find any external IPs, it falls back to
+// looking for internal IPs. If it can't find an internal IP for every node it
+// returns an error, though it still returns all hosts that it found in that
+// case.
+func NodeSSHHosts(c clientset.Interface) ([]string, error) {
+	nodelist := waitListSchedulableNodesOrDie(c)
+
+	hosts := NodeAddresses(nodelist, v1.NodeExternalIP)
+	// If ExternalIPs aren't set, assume the test programs can reach the
+	// InternalIP. Simplified exception logic here assumes that the hosts will
+	// either all have ExternalIP or none will. Simplifies handling here and
+	// should be adequate since the setting of the external IPs is provider
+	// specific: they should either all have them or none of them will.
+	if len(hosts) == 0 {
+		Logf("No external IP address on nodes, falling back to internal IPs")
+		hosts = NodeAddresses(nodelist, v1.NodeInternalIP)
+	}
+
+	// Error if any node didn't have an external/internal IP.
+	if len(hosts) != len(nodelist.Items) {
+		return hosts, fmt.Errorf(
+			"only found %d IPs on nodes, but found %d nodes. Nodelist: %v",
+			len(hosts), len(nodelist.Items), nodelist)
+	}
+
+	sshHosts := make([]string, 0, len(hosts))
+	for _, h := range hosts {
+		sshHosts = append(sshHosts, net.JoinHostPort(h, sshPort))
+	}
+	return sshHosts, nil
+}
+
+type SSHResult struct {
+	User   string
+	Host   string
+	Cmd    string
+	Stdout string
+	Stderr string
+	Code   int
+}
+
+// NodeExec execs the given cmd on node via SSH. Note that the nodeName is an sshable name,
+// eg: the name returned by framework.GetMasterHost(). This is also not guaranteed to work across
+// cloud providers since it involves ssh.
+func NodeExec(nodeName, cmd string) (SSHResult, error) {
+	return SSH(cmd, net.JoinHostPort(nodeName, sshPort), TestContext.Provider)
+}
+
+// SSH synchronously SSHs to a node running on provider and runs cmd. If there
+// is no error performing the SSH, the stdout, stderr, and exit code are
+// returned.
+func SSH(cmd, host, provider string) (SSHResult, error) {
+	result := SSHResult{Host: host, Cmd: cmd}
+
+	// Get a signer for the provider.
+	signer, err := GetSigner(provider)
+	if err != nil {
+		return result, fmt.Errorf("error getting signer for provider %s: '%v'", provider, err)
+	}
+
+	// RunSSHCommand will default to Getenv("USER") if user == "", but we're
+	// defaulting here as well for logging clarity.
+	result.User = os.Getenv("KUBE_SSH_USER")
+	if result.User == "" {
+		result.User = os.Getenv("USER")
+	}
+
+	if bastion := os.Getenv("KUBE_SSH_BASTION"); len(bastion) > 0 {
+		stdout, stderr, code, err := RunSSHCommandViaBastion(cmd, result.User, bastion, host, signer)
+		result.Stdout = stdout
+		result.Stderr = stderr
+		result.Code = code
+		return result, err
+	}
+
+	stdout, stderr, code, err := sshutil.RunSSHCommand(cmd, result.User, host, signer)
+	result.Stdout = stdout
+	result.Stderr = stderr
+	result.Code = code
+
+	return result, err
+}
+
+// RunSSHCommandViaBastion returns the stdout, stderr, and exit code from running cmd on
+// host as specific user, along with any SSH-level error. It uses an SSH proxy to connect
+// to bastion, then via that tunnel connects to the remote host. Similar to
+// sshutil.RunSSHCommand but scoped to the needs of the test infrastructure.
+func RunSSHCommandViaBastion(cmd, user, bastion, host string, signer ssh.Signer) (string, string, int, error) {
+	// Setup the config, dial the server, and open a session.
+	config := &ssh.ClientConfig{
+		User:            user,
+		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         150 * time.Second,
+	}
+	bastionClient, err := ssh.Dial("tcp", bastion, config)
+	if err != nil {
+		err = wait.Poll(5*time.Second, 20*time.Second, func() (bool, error) {
+			fmt.Printf("error dialing %s@%s: '%v', retrying\n", user, bastion, err)
+			if bastionClient, err = ssh.Dial("tcp", bastion, config); err != nil {
+				return false, err
+			}
+			return true, nil
+		})
+	}
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error getting SSH client to %s@%s: %v", user, bastion, err)
+	}
+	defer bastionClient.Close()
+
+	conn, err := bastionClient.Dial("tcp", host)
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error dialing %s from bastion: %v", host, err)
+	}
+	defer conn.Close()
+
+	ncc, chans, reqs, err := ssh.NewClientConn(conn, host, config)
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error creating forwarding connection %s from bastion: %v", host, err)
+	}
+	client := ssh.NewClient(ncc, chans, reqs)
+	defer client.Close()
+
+	session, err := client.NewSession()
+	if err != nil {
+		return "", "", 0, fmt.Errorf("error creating session to %s@%s from bastion: '%v'", user, host, err)
+	}
+	defer session.Close()
+
+	// Run the command.
+	code := 0
+	var bout, berr bytes.Buffer
+	session.Stdout, session.Stderr = &bout, &berr
+	if err = session.Run(cmd); err != nil {
+		// Check whether the command failed to run or didn't complete.
+		if exiterr, ok := err.(*ssh.ExitError); ok {
+			// If we got an ExitError and the exit code is nonzero, we'll
+			// consider the SSH itself successful (just that the command run
+			// errored on the host).
+			if code = exiterr.ExitStatus(); code != 0 {
+				err = nil
+			}
+		} else {
+			// Some other kind of error happened (e.g. an IOError); consider the
+			// SSH unsuccessful.
+			err = fmt.Errorf("failed running `%s` on %s@%s: '%v'", cmd, user, host, err)
+		}
+	}
+	return bout.String(), berr.String(), code, err
+}
+
+func LogSSHResult(result SSHResult) {
+	remote := fmt.Sprintf("%s@%s", result.User, result.Host)
+	Logf("ssh %s: command:   %s", remote, result.Cmd)
+	Logf("ssh %s: stdout:    %q", remote, result.Stdout)
+	Logf("ssh %s: stderr:    %q", remote, result.Stderr)
+	Logf("ssh %s: exit code: %d", remote, result.Code)
+}
+
+func IssueSSHCommandWithResult(cmd, provider string, node *v1.Node) (*SSHResult, error) {
+	Logf("Getting external IP address for %s", node.Name)
+	host := ""
+	for _, a := range node.Status.Addresses {
+		if a.Type == v1.NodeExternalIP {
+			host = net.JoinHostPort(a.Address, sshPort)
+			break
+		}
+	}
+
+	if host == "" {
+		// No external IPs were found, let's try to use internal as plan B
+		for _, a := range node.Status.Addresses {
+			if a.Type == v1.NodeInternalIP {
+				host = net.JoinHostPort(a.Address, sshPort)
+				break
+			}
+		}
+	}
+
+	if host == "" {
+		return nil, fmt.Errorf("couldn't find any IP address for node %s", node.Name)
+	}
+
+	Logf("SSH %q on %s(%s)", cmd, node.Name, host)
+	result, err := SSH(cmd, host, provider)
+	LogSSHResult(result)
+
+	if result.Code != 0 || err != nil {
+		return nil, fmt.Errorf("failed running %q: %v (exit code %d, stderr %v)",
+			cmd, err, result.Code, result.Stderr)
+	}
+
+	return &result, nil
+}
+
+func IssueSSHCommand(cmd, provider string, node *v1.Node) error {
+	_, err := IssueSSHCommandWithResult(cmd, provider, node)
+	if err != nil {
+		return err
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/statefulset_utils.go

@@ -0,0 +1,892 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"path/filepath"
+	"reflect"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	apps "k8s.io/api/apps/v1"
+	appsV1beta2 "k8s.io/api/apps/v1beta2"
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
+	"k8s.io/kubernetes/test/e2e/manifest"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+)
+
+const (
+	// Poll interval for StatefulSet tests
+	StatefulSetPoll = 10 * time.Second
+	// Timeout interval for StatefulSet operations
+	StatefulSetTimeout = 10 * time.Minute
+	// Timeout for stateful pods to change state
+	StatefulPodTimeout = 5 * time.Minute
+)
+
+// CreateStatefulSetService creates a Headless Service with Name name and Selector set to match labels.
+func CreateStatefulSetService(name string, labels map[string]string) *v1.Service {
+	headlessService := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: labels,
+		},
+	}
+	headlessService.Spec.Ports = []v1.ServicePort{
+		{Port: 80, Name: "http", Protocol: v1.ProtocolTCP},
+	}
+	headlessService.Spec.ClusterIP = "None"
+	return headlessService
+}
+
+// StatefulSetTester is a struct that contains utility methods for testing StatefulSet related functionality. It uses a
+// clientset.Interface to communicate with the API server.
+type StatefulSetTester struct {
+	c clientset.Interface
+}
+
+// NewStatefulSetTester creates a StatefulSetTester that uses c to interact with the API server.
+func NewStatefulSetTester(c clientset.Interface) *StatefulSetTester {
+	return &StatefulSetTester{c}
+}
+
+// GetStatefulSet gets the StatefulSet named name in namespace.
+func (s *StatefulSetTester) GetStatefulSet(namespace, name string) *apps.StatefulSet {
+	ss, err := s.c.AppsV1().StatefulSets(namespace).Get(name, metav1.GetOptions{})
+	if err != nil {
+		Failf("Failed to get StatefulSet %s/%s: %v", namespace, name, err)
+	}
+	return ss
+}
+
+// CreateStatefulSet creates a StatefulSet from the manifest at manifestPath in the Namespace ns using kubectl create.
+func (s *StatefulSetTester) CreateStatefulSet(manifestPath, ns string) *apps.StatefulSet {
+	mkpath := func(file string) string {
+		return filepath.Join(manifestPath, file)
+	}
+
+	Logf("Parsing statefulset from %v", mkpath("statefulset.yaml"))
+	ss, err := manifest.StatefulSetFromManifest(mkpath("statefulset.yaml"), ns)
+	ExpectNoError(err)
+	Logf("Parsing service from %v", mkpath("service.yaml"))
+	svc, err := manifest.SvcFromManifest(mkpath("service.yaml"))
+	ExpectNoError(err)
+
+	Logf(fmt.Sprintf("creating " + ss.Name + " service"))
+	_, err = s.c.CoreV1().Services(ns).Create(svc)
+	ExpectNoError(err)
+
+	Logf(fmt.Sprintf("creating statefulset %v/%v with %d replicas and selector %+v", ss.Namespace, ss.Name, *(ss.Spec.Replicas), ss.Spec.Selector))
+	_, err = s.c.AppsV1().StatefulSets(ns).Create(ss)
+	ExpectNoError(err)
+	s.WaitForRunningAndReady(*ss.Spec.Replicas, ss)
+	return ss
+}
+
+// CheckMount checks that the mount at mountPath is valid for all Pods in ss.
+func (s *StatefulSetTester) CheckMount(ss *apps.StatefulSet, mountPath string) error {
+	for _, cmd := range []string{
+		// Print inode, size etc
+		fmt.Sprintf("ls -idlh %v", mountPath),
+		// Print subdirs
+		fmt.Sprintf("find %v", mountPath),
+		// Try writing
+		fmt.Sprintf("touch %v", filepath.Join(mountPath, fmt.Sprintf("%v", time.Now().UnixNano()))),
+	} {
+		if err := s.ExecInStatefulPods(ss, cmd); err != nil {
+			return fmt.Errorf("failed to execute %v, error: %v", cmd, err)
+		}
+	}
+	return nil
+}
+
+// ExecInStatefulPods executes cmd in all Pods in ss. If a error occurs it is returned and cmd is not execute in any subsequent Pods.
+func (s *StatefulSetTester) ExecInStatefulPods(ss *apps.StatefulSet, cmd string) error {
+	podList := s.GetPodList(ss)
+	for _, statefulPod := range podList.Items {
+		stdout, err := RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
+		Logf("stdout of %v on %v: %v", cmd, statefulPod.Name, stdout)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CheckHostname verifies that all Pods in ss have the correct Hostname. If the returned error is not nil than verification failed.
+func (s *StatefulSetTester) CheckHostname(ss *apps.StatefulSet) error {
+	cmd := "printf $(hostname)"
+	podList := s.GetPodList(ss)
+	for _, statefulPod := range podList.Items {
+		hostname, err := RunHostCmdWithRetries(statefulPod.Namespace, statefulPod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
+		if err != nil {
+			return err
+		}
+		if hostname != statefulPod.Name {
+			return fmt.Errorf("unexpected hostname (%s) and stateful pod name (%s) not equal", hostname, statefulPod.Name)
+		}
+	}
+	return nil
+}
+
+// Saturate waits for all Pods in ss to become Running and Ready.
+func (s *StatefulSetTester) Saturate(ss *apps.StatefulSet) {
+	var i int32
+	for i = 0; i < *(ss.Spec.Replicas); i++ {
+		Logf("Waiting for stateful pod at index %v to enter Running", i)
+		s.WaitForRunning(i+1, i, ss)
+		Logf("Resuming stateful pod at index %v", i)
+		s.ResumeNextPod(ss)
+	}
+}
+
+// DeleteStatefulPodAtIndex deletes the Pod with ordinal index in ss.
+func (s *StatefulSetTester) DeleteStatefulPodAtIndex(index int, ss *apps.StatefulSet) {
+	name := getStatefulSetPodNameAtIndex(index, ss)
+	noGrace := int64(0)
+	if err := s.c.CoreV1().Pods(ss.Namespace).Delete(name, &metav1.DeleteOptions{GracePeriodSeconds: &noGrace}); err != nil {
+		Failf("Failed to delete stateful pod %v for StatefulSet %v/%v: %v", name, ss.Namespace, ss.Name, err)
+	}
+}
+
+// VerifyStatefulPodFunc is a func that examines a StatefulSetPod.
+type VerifyStatefulPodFunc func(*v1.Pod)
+
+// VerifyPodAtIndex applies a visitor patter to the Pod at index in ss. verify is applied to the Pod to "visit" it.
+func (s *StatefulSetTester) VerifyPodAtIndex(index int, ss *apps.StatefulSet, verify VerifyStatefulPodFunc) {
+	name := getStatefulSetPodNameAtIndex(index, ss)
+	pod, err := s.c.CoreV1().Pods(ss.Namespace).Get(name, metav1.GetOptions{})
+	ExpectNoError(err, fmt.Sprintf("Failed to get stateful pod %s for StatefulSet %s/%s", name, ss.Namespace, ss.Name))
+	verify(pod)
+}
+
+func getStatefulSetPodNameAtIndex(index int, ss *apps.StatefulSet) string {
+	// TODO: we won't use "-index" as the name strategy forever,
+	// pull the name out from an identity mapper.
+	return fmt.Sprintf("%v-%v", ss.Name, index)
+}
+
+// Scale scales ss to count replicas.
+func (s *StatefulSetTester) Scale(ss *apps.StatefulSet, count int32) (*apps.StatefulSet, error) {
+	name := ss.Name
+	ns := ss.Namespace
+
+	Logf("Scaling statefulset %s to %d", name, count)
+	ss = s.update(ns, name, func(ss *apps.StatefulSet) { *(ss.Spec.Replicas) = count })
+
+	var statefulPodList *v1.PodList
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, func() (bool, error) {
+		statefulPodList = s.GetPodList(ss)
+		if int32(len(statefulPodList.Items)) == count {
+			return true, nil
+		}
+		return false, nil
+	})
+	if pollErr != nil {
+		unhealthy := []string{}
+		for _, statefulPod := range statefulPodList.Items {
+			delTs, phase, readiness := statefulPod.DeletionTimestamp, statefulPod.Status.Phase, podutil.IsPodReady(&statefulPod)
+			if delTs != nil || phase != v1.PodRunning || !readiness {
+				unhealthy = append(unhealthy, fmt.Sprintf("%v: deletion %v, phase %v, readiness %v", statefulPod.Name, delTs, phase, readiness))
+			}
+		}
+		return ss, fmt.Errorf("Failed to scale statefulset to %d in %v. Remaining pods:\n%v", count, StatefulSetTimeout, unhealthy)
+	}
+	return ss, nil
+}
+
+// UpdateReplicas updates the replicas of ss to count.
+func (s *StatefulSetTester) UpdateReplicas(ss *apps.StatefulSet, count int32) {
+	s.update(ss.Namespace, ss.Name, func(ss *apps.StatefulSet) { *(ss.Spec.Replicas) = count })
+}
+
+// Restart scales ss to 0 and then back to its previous number of replicas.
+func (s *StatefulSetTester) Restart(ss *apps.StatefulSet) {
+	oldReplicas := *(ss.Spec.Replicas)
+	ss, err := s.Scale(ss, 0)
+	ExpectNoError(err)
+	// Wait for controller to report the desired number of Pods.
+	// This way we know the controller has observed all Pod deletions
+	// before we scale it back up.
+	s.WaitForStatusReplicas(ss, 0)
+	s.update(ss.Namespace, ss.Name, func(ss *apps.StatefulSet) { *(ss.Spec.Replicas) = oldReplicas })
+}
+
+func (s *StatefulSetTester) update(ns, name string, update func(ss *apps.StatefulSet)) *apps.StatefulSet {
+	for i := 0; i < 3; i++ {
+		ss, err := s.c.AppsV1().StatefulSets(ns).Get(name, metav1.GetOptions{})
+		if err != nil {
+			Failf("failed to get statefulset %q: %v", name, err)
+		}
+		update(ss)
+		ss, err = s.c.AppsV1().StatefulSets(ns).Update(ss)
+		if err == nil {
+			return ss
+		}
+		if !apierrs.IsConflict(err) && !apierrs.IsServerTimeout(err) {
+			Failf("failed to update statefulset %q: %v", name, err)
+		}
+	}
+	Failf("too many retries draining statefulset %q", name)
+	return nil
+}
+
+// GetPodList gets the current Pods in ss.
+func (s *StatefulSetTester) GetPodList(ss *apps.StatefulSet) *v1.PodList {
+	selector, err := metav1.LabelSelectorAsSelector(ss.Spec.Selector)
+	ExpectNoError(err)
+	podList, err := s.c.CoreV1().Pods(ss.Namespace).List(metav1.ListOptions{LabelSelector: selector.String()})
+	ExpectNoError(err)
+	return podList
+}
+
+// ConfirmStatefulPodCount asserts that the current number of Pods in ss is count waiting up to timeout for ss to
+// to scale to count.
+func (s *StatefulSetTester) ConfirmStatefulPodCount(count int, ss *apps.StatefulSet, timeout time.Duration, hard bool) {
+	start := time.Now()
+	deadline := start.Add(timeout)
+	for t := time.Now(); t.Before(deadline); t = time.Now() {
+		podList := s.GetPodList(ss)
+		statefulPodCount := len(podList.Items)
+		if statefulPodCount != count {
+			logPodStates(podList.Items)
+			if hard {
+				Failf("StatefulSet %v scaled unexpectedly scaled to %d -> %d replicas", ss.Name, count, len(podList.Items))
+			} else {
+				Logf("StatefulSet %v has not reached scale %d, at %d", ss.Name, count, statefulPodCount)
+			}
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		Logf("Verifying statefulset %v doesn't scale past %d for another %+v", ss.Name, count, deadline.Sub(t))
+		time.Sleep(1 * time.Second)
+	}
+}
+
+// WaitForRunning waits for numPodsRunning in ss to be Running and for the first
+// numPodsReady ordinals to be Ready.
+func (s *StatefulSetTester) WaitForRunning(numPodsRunning, numPodsReady int32, ss *apps.StatefulSet) {
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout,
+		func() (bool, error) {
+			podList := s.GetPodList(ss)
+			s.SortStatefulPods(podList)
+			if int32(len(podList.Items)) < numPodsRunning {
+				Logf("Found %d stateful pods, waiting for %d", len(podList.Items), numPodsRunning)
+				return false, nil
+			}
+			if int32(len(podList.Items)) > numPodsRunning {
+				return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numPodsRunning, len(podList.Items))
+			}
+			for _, p := range podList.Items {
+				shouldBeReady := getStatefulPodOrdinal(&p) < int(numPodsReady)
+				isReady := podutil.IsPodReady(&p)
+				desiredReadiness := shouldBeReady == isReady
+				Logf("Waiting for pod %v to enter %v - Ready=%v, currently %v - Ready=%v", p.Name, v1.PodRunning, shouldBeReady, p.Status.Phase, isReady)
+				if p.Status.Phase != v1.PodRunning || !desiredReadiness {
+					return false, nil
+				}
+			}
+			return true, nil
+		})
+	if pollErr != nil {
+		Failf("Failed waiting for pods to enter running: %v", pollErr)
+	}
+}
+
+// WaitForState periodically polls for the ss and its pods until the until function returns either true or an error
+func (s *StatefulSetTester) WaitForState(ss *apps.StatefulSet, until func(*apps.StatefulSet, *v1.PodList) (bool, error)) {
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout,
+		func() (bool, error) {
+			ssGet, err := s.c.AppsV1().StatefulSets(ss.Namespace).Get(ss.Name, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+			podList := s.GetPodList(ssGet)
+			return until(ssGet, podList)
+		})
+	if pollErr != nil {
+		Failf("Failed waiting for state update: %v", pollErr)
+	}
+}
+
+// WaitForStatus waits for the StatefulSetStatus's ObservedGeneration to be greater than or equal to set's Generation.
+// The returned StatefulSet contains such a StatefulSetStatus
+func (s *StatefulSetTester) WaitForStatus(set *apps.StatefulSet) *apps.StatefulSet {
+	s.WaitForState(set, func(set2 *apps.StatefulSet, pods *v1.PodList) (bool, error) {
+		if set2.Status.ObservedGeneration >= set.Generation {
+			set = set2
+			return true, nil
+		}
+		return false, nil
+	})
+	return set
+}
+
+// WaitForRunningAndReady waits for numStatefulPods in ss to be Running and Ready.
+func (s *StatefulSetTester) WaitForRunningAndReady(numStatefulPods int32, ss *apps.StatefulSet) {
+	s.WaitForRunning(numStatefulPods, numStatefulPods, ss)
+}
+
+// WaitForPodReady waits for the Pod named podName in set to exist and have a Ready condition.
+func (s *StatefulSetTester) WaitForPodReady(set *apps.StatefulSet, podName string) (*apps.StatefulSet, *v1.PodList) {
+	var pods *v1.PodList
+	s.WaitForState(set, func(set2 *apps.StatefulSet, pods2 *v1.PodList) (bool, error) {
+		set = set2
+		pods = pods2
+		for i := range pods.Items {
+			if pods.Items[i].Name == podName {
+				return podutil.IsPodReady(&pods.Items[i]), nil
+			}
+		}
+		return false, nil
+	})
+	return set, pods
+
+}
+
+// WaitForPodNotReady waist for the Pod named podName in set to exist and to not have a Ready condition.
+func (s *StatefulSetTester) WaitForPodNotReady(set *apps.StatefulSet, podName string) (*apps.StatefulSet, *v1.PodList) {
+	var pods *v1.PodList
+	s.WaitForState(set, func(set2 *apps.StatefulSet, pods2 *v1.PodList) (bool, error) {
+		set = set2
+		pods = pods2
+		for i := range pods.Items {
+			if pods.Items[i].Name == podName {
+				return !podutil.IsPodReady(&pods.Items[i]), nil
+			}
+		}
+		return false, nil
+	})
+	return set, pods
+
+}
+
+// WaitForRollingUpdate waits for all Pods in set to exist and have the correct revision and for the RollingUpdate to
+// complete. set must have a RollingUpdateStatefulSetStrategyType.
+func (s *StatefulSetTester) WaitForRollingUpdate(set *apps.StatefulSet) (*apps.StatefulSet, *v1.PodList) {
+	var pods *v1.PodList
+	if set.Spec.UpdateStrategy.Type != apps.RollingUpdateStatefulSetStrategyType {
+		Failf("StatefulSet %s/%s attempt to wait for rolling update with updateStrategy %s",
+			set.Namespace,
+			set.Name,
+			set.Spec.UpdateStrategy.Type)
+	}
+	s.WaitForState(set, func(set2 *apps.StatefulSet, pods2 *v1.PodList) (bool, error) {
+		set = set2
+		pods = pods2
+		if len(pods.Items) < int(*set.Spec.Replicas) {
+			return false, nil
+		}
+		if set.Status.UpdateRevision != set.Status.CurrentRevision {
+			Logf("Waiting for StatefulSet %s/%s to complete update",
+				set.Namespace,
+				set.Name,
+			)
+			s.SortStatefulPods(pods)
+			for i := range pods.Items {
+				if pods.Items[i].Labels[apps.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
+					Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
+						pods.Items[i].Namespace,
+						pods.Items[i].Name,
+						set.Status.UpdateRevision,
+						pods.Items[i].Labels[apps.StatefulSetRevisionLabel])
+				}
+			}
+			return false, nil
+		}
+		return true, nil
+	})
+	return set, pods
+}
+
+// WaitForPartitionedRollingUpdate waits for all Pods in set to exist and have the correct revision. set must have
+// a RollingUpdateStatefulSetStrategyType with a non-nil RollingUpdate and Partition. All Pods with ordinals less
+// than or equal to the Partition are expected to be at set's current revision. All other Pods are expected to be
+// at its update revision.
+func (s *StatefulSetTester) WaitForPartitionedRollingUpdate(set *apps.StatefulSet) (*apps.StatefulSet, *v1.PodList) {
+	var pods *v1.PodList
+	if set.Spec.UpdateStrategy.Type != apps.RollingUpdateStatefulSetStrategyType {
+		Failf("StatefulSet %s/%s attempt to wait for partitioned update with updateStrategy %s",
+			set.Namespace,
+			set.Name,
+			set.Spec.UpdateStrategy.Type)
+	}
+	if set.Spec.UpdateStrategy.RollingUpdate == nil || set.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
+		Failf("StatefulSet %s/%s attempt to wait for partitioned update with nil RollingUpdate or nil Partition",
+			set.Namespace,
+			set.Name)
+	}
+	s.WaitForState(set, func(set2 *apps.StatefulSet, pods2 *v1.PodList) (bool, error) {
+		set = set2
+		pods = pods2
+		partition := int(*set.Spec.UpdateStrategy.RollingUpdate.Partition)
+		if len(pods.Items) < int(*set.Spec.Replicas) {
+			return false, nil
+		}
+		if partition <= 0 && set.Status.UpdateRevision != set.Status.CurrentRevision {
+			Logf("Waiting for StatefulSet %s/%s to complete update",
+				set.Namespace,
+				set.Name,
+			)
+			s.SortStatefulPods(pods)
+			for i := range pods.Items {
+				if pods.Items[i].Labels[apps.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
+					Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
+						pods.Items[i].Namespace,
+						pods.Items[i].Name,
+						set.Status.UpdateRevision,
+						pods.Items[i].Labels[apps.StatefulSetRevisionLabel])
+				}
+			}
+			return false, nil
+		} else {
+			for i := int(*set.Spec.Replicas) - 1; i >= partition; i-- {
+				if pods.Items[i].Labels[apps.StatefulSetRevisionLabel] != set.Status.UpdateRevision {
+					Logf("Waiting for Pod %s/%s to have revision %s update revision %s",
+						pods.Items[i].Namespace,
+						pods.Items[i].Name,
+						set.Status.UpdateRevision,
+						pods.Items[i].Labels[apps.StatefulSetRevisionLabel])
+					return false, nil
+				}
+			}
+		}
+		return true, nil
+	})
+	return set, pods
+}
+
+// WaitForRunningAndReady waits for numStatefulPods in ss to be Running and not Ready.
+func (s *StatefulSetTester) WaitForRunningAndNotReady(numStatefulPods int32, ss *apps.StatefulSet) {
+	s.WaitForRunning(numStatefulPods, 0, ss)
+}
+
+var httpProbe = &v1.Probe{
+	Handler: v1.Handler{
+		HTTPGet: &v1.HTTPGetAction{
+			Path: "/index.html",
+			Port: intstr.IntOrString{IntVal: 80},
+		},
+	},
+	PeriodSeconds:    1,
+	SuccessThreshold: 1,
+	FailureThreshold: 1,
+}
+
+// SetHttpProbe sets the pod template's ReadinessProbe for Nginx StatefulSet containers.
+// This probe can then be controlled with BreakHttpProbe() and RestoreHttpProbe().
+// Note that this cannot be used together with PauseNewPods().
+func (s *StatefulSetTester) SetHttpProbe(ss *apps.StatefulSet) {
+	ss.Spec.Template.Spec.Containers[0].ReadinessProbe = httpProbe
+}
+
+// BreakHttpProbe breaks the readiness probe for Nginx StatefulSet containers in ss.
+func (s *StatefulSetTester) BreakHttpProbe(ss *apps.StatefulSet) error {
+	path := httpProbe.HTTPGet.Path
+	if path == "" {
+		return fmt.Errorf("Path expected to be not empty: %v", path)
+	}
+	// Ignore 'mv' errors to make this idempotent.
+	cmd := fmt.Sprintf("mv -v /usr/share/nginx/html%v /tmp/ || true", path)
+	return s.ExecInStatefulPods(ss, cmd)
+}
+
+// BreakPodHttpProbe breaks the readiness probe for Nginx StatefulSet containers in one pod.
+func (s *StatefulSetTester) BreakPodHttpProbe(ss *apps.StatefulSet, pod *v1.Pod) error {
+	path := httpProbe.HTTPGet.Path
+	if path == "" {
+		return fmt.Errorf("Path expected to be not empty: %v", path)
+	}
+	// Ignore 'mv' errors to make this idempotent.
+	cmd := fmt.Sprintf("mv -v /usr/share/nginx/html%v /tmp/ || true", path)
+	stdout, err := RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
+	Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
+	return err
+}
+
+// RestoreHttpProbe restores the readiness probe for Nginx StatefulSet containers in ss.
+func (s *StatefulSetTester) RestoreHttpProbe(ss *apps.StatefulSet) error {
+	path := httpProbe.HTTPGet.Path
+	if path == "" {
+		return fmt.Errorf("Path expected to be not empty: %v", path)
+	}
+	// Ignore 'mv' errors to make this idempotent.
+	cmd := fmt.Sprintf("mv -v /tmp%v /usr/share/nginx/html/ || true", path)
+	return s.ExecInStatefulPods(ss, cmd)
+}
+
+// RestorePodHttpProbe restores the readiness probe for Nginx StatefulSet containers in pod.
+func (s *StatefulSetTester) RestorePodHttpProbe(ss *apps.StatefulSet, pod *v1.Pod) error {
+	path := httpProbe.HTTPGet.Path
+	if path == "" {
+		return fmt.Errorf("Path expected to be not empty: %v", path)
+	}
+	// Ignore 'mv' errors to make this idempotent.
+	cmd := fmt.Sprintf("mv -v /tmp%v /usr/share/nginx/html/ || true", path)
+	stdout, err := RunHostCmdWithRetries(pod.Namespace, pod.Name, cmd, StatefulSetPoll, StatefulPodTimeout)
+	Logf("stdout of %v on %v: %v", cmd, pod.Name, stdout)
+	return err
+}
+
+var pauseProbe = &v1.Probe{
+	Handler: v1.Handler{
+		Exec: &v1.ExecAction{Command: []string{"test", "-f", "/data/statefulset-continue"}},
+	},
+	PeriodSeconds:    1,
+	SuccessThreshold: 1,
+	FailureThreshold: 1,
+}
+
+func hasPauseProbe(pod *v1.Pod) bool {
+	probe := pod.Spec.Containers[0].ReadinessProbe
+	return probe != nil && reflect.DeepEqual(probe.Exec.Command, pauseProbe.Exec.Command)
+}
+
+// PauseNewPods adds an always-failing ReadinessProbe to the StatefulSet PodTemplate.
+// This causes all newly-created Pods to stay Unready until they are manually resumed
+// with ResumeNextPod().
+// Note that this cannot be used together with SetHttpProbe().
+func (s *StatefulSetTester) PauseNewPods(ss *apps.StatefulSet) {
+	ss.Spec.Template.Spec.Containers[0].ReadinessProbe = pauseProbe
+}
+
+// ResumeNextPod allows the next Pod in the StatefulSet to continue by removing the ReadinessProbe
+// added by PauseNewPods(), if it's still there.
+// It fails the test if it finds any pods that are not in phase Running,
+// or if it finds more than one paused Pod existing at the same time.
+// This is a no-op if there are no paused pods.
+func (s *StatefulSetTester) ResumeNextPod(ss *apps.StatefulSet) {
+	podList := s.GetPodList(ss)
+	resumedPod := ""
+	for _, pod := range podList.Items {
+		if pod.Status.Phase != v1.PodRunning {
+			Failf("Found pod in phase %q, cannot resume", pod.Status.Phase)
+		}
+		if podutil.IsPodReady(&pod) || !hasPauseProbe(&pod) {
+			continue
+		}
+		if resumedPod != "" {
+			Failf("Found multiple paused stateful pods: %v and %v", pod.Name, resumedPod)
+		}
+		_, err := RunHostCmdWithRetries(pod.Namespace, pod.Name, "dd if=/dev/zero of=/data/statefulset-continue bs=1 count=1 conv=fsync", StatefulSetPoll, StatefulPodTimeout)
+		ExpectNoError(err)
+		Logf("Resumed pod %v", pod.Name)
+		resumedPod = pod.Name
+	}
+}
+
+// WaitForStatusReadyReplicas waits for the ss.Status.ReadyReplicas to be equal to expectedReplicas
+func (s *StatefulSetTester) WaitForStatusReadyReplicas(ss *apps.StatefulSet, expectedReplicas int32) {
+	Logf("Waiting for statefulset status.replicas updated to %d", expectedReplicas)
+
+	ns, name := ss.Namespace, ss.Name
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout,
+		func() (bool, error) {
+			ssGet, err := s.c.AppsV1().StatefulSets(ns).Get(name, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+			if ssGet.Status.ObservedGeneration < ss.Generation {
+				return false, nil
+			}
+			if ssGet.Status.ReadyReplicas != expectedReplicas {
+				Logf("Waiting for stateful set status.readyReplicas to become %d, currently %d", expectedReplicas, ssGet.Status.ReadyReplicas)
+				return false, nil
+			}
+			return true, nil
+		})
+	if pollErr != nil {
+		Failf("Failed waiting for stateful set status.readyReplicas updated to %d: %v", expectedReplicas, pollErr)
+	}
+}
+
+// WaitForStatusReplicas waits for the ss.Status.Replicas to be equal to expectedReplicas
+func (s *StatefulSetTester) WaitForStatusReplicas(ss *apps.StatefulSet, expectedReplicas int32) {
+	Logf("Waiting for statefulset status.replicas updated to %d", expectedReplicas)
+
+	ns, name := ss.Namespace, ss.Name
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout,
+		func() (bool, error) {
+			ssGet, err := s.c.AppsV1().StatefulSets(ns).Get(name, metav1.GetOptions{})
+			if err != nil {
+				return false, err
+			}
+			if ssGet.Status.ObservedGeneration < ss.Generation {
+				return false, nil
+			}
+			if ssGet.Status.Replicas != expectedReplicas {
+				Logf("Waiting for stateful set status.replicas to become %d, currently %d", expectedReplicas, ssGet.Status.Replicas)
+				return false, nil
+			}
+			return true, nil
+		})
+	if pollErr != nil {
+		Failf("Failed waiting for stateful set status.replicas updated to %d: %v", expectedReplicas, pollErr)
+	}
+}
+
+// CheckServiceName asserts that the ServiceName for ss is equivalent to expectedServiceName.
+func (p *StatefulSetTester) CheckServiceName(ss *apps.StatefulSet, expectedServiceName string) error {
+	Logf("Checking if statefulset spec.serviceName is %s", expectedServiceName)
+
+	if expectedServiceName != ss.Spec.ServiceName {
+		return fmt.Errorf("Wrong service name governing statefulset. Expected %s got %s",
+			expectedServiceName, ss.Spec.ServiceName)
+	}
+
+	return nil
+}
+
+// SortStatefulPods sorts pods by their ordinals
+func (s *StatefulSetTester) SortStatefulPods(pods *v1.PodList) {
+	sort.Sort(statefulPodsByOrdinal(pods.Items))
+}
+
+// DeleteAllStatefulSets deletes all StatefulSet API Objects in Namespace ns.
+func DeleteAllStatefulSets(c clientset.Interface, ns string) {
+	sst := &StatefulSetTester{c: c}
+	ssList, err := c.AppsV1().StatefulSets(ns).List(metav1.ListOptions{LabelSelector: labels.Everything().String()})
+	ExpectNoError(err)
+
+	// Scale down each statefulset, then delete it completely.
+	// Deleting a pvc without doing this will leak volumes, #25101.
+	errList := []string{}
+	for i := range ssList.Items {
+		ss := &ssList.Items[i]
+		var err error
+		if ss, err = sst.Scale(ss, 0); err != nil {
+			errList = append(errList, fmt.Sprintf("%v", err))
+		}
+		sst.WaitForStatusReplicas(ss, 0)
+		Logf("Deleting statefulset %v", ss.Name)
+		// Use OrphanDependents=false so it's deleted synchronously.
+		// We already made sure the Pods are gone inside Scale().
+		if err := c.AppsV1().StatefulSets(ss.Namespace).Delete(ss.Name, &metav1.DeleteOptions{OrphanDependents: new(bool)}); err != nil {
+			errList = append(errList, fmt.Sprintf("%v", err))
+		}
+	}
+
+	// pvs are global, so we need to wait for the exact ones bound to the statefulset pvcs.
+	pvNames := sets.NewString()
+	// TODO: Don't assume all pvcs in the ns belong to a statefulset
+	pvcPollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, func() (bool, error) {
+		pvcList, err := c.CoreV1().PersistentVolumeClaims(ns).List(metav1.ListOptions{LabelSelector: labels.Everything().String()})
+		if err != nil {
+			Logf("WARNING: Failed to list pvcs, retrying %v", err)
+			return false, nil
+		}
+		for _, pvc := range pvcList.Items {
+			pvNames.Insert(pvc.Spec.VolumeName)
+			// TODO: Double check that there are no pods referencing the pvc
+			Logf("Deleting pvc: %v with volume %v", pvc.Name, pvc.Spec.VolumeName)
+			if err := c.CoreV1().PersistentVolumeClaims(ns).Delete(pvc.Name, nil); err != nil {
+				return false, nil
+			}
+		}
+		return true, nil
+	})
+	if pvcPollErr != nil {
+		errList = append(errList, fmt.Sprintf("Timeout waiting for pvc deletion."))
+	}
+
+	pollErr := wait.PollImmediate(StatefulSetPoll, StatefulSetTimeout, func() (bool, error) {
+		pvList, err := c.CoreV1().PersistentVolumes().List(metav1.ListOptions{LabelSelector: labels.Everything().String()})
+		if err != nil {
+			Logf("WARNING: Failed to list pvs, retrying %v", err)
+			return false, nil
+		}
+		waitingFor := []string{}
+		for _, pv := range pvList.Items {
+			if pvNames.Has(pv.Name) {
+				waitingFor = append(waitingFor, fmt.Sprintf("%v: %+v", pv.Name, pv.Status))
+			}
+		}
+		if len(waitingFor) == 0 {
+			return true, nil
+		}
+		Logf("Still waiting for pvs of statefulset to disappear:\n%v", strings.Join(waitingFor, "\n"))
+		return false, nil
+	})
+	if pollErr != nil {
+		errList = append(errList, fmt.Sprintf("Timeout waiting for pv provisioner to delete pvs, this might mean the test leaked pvs."))
+	}
+	if len(errList) != 0 {
+		ExpectNoError(fmt.Errorf("%v", strings.Join(errList, "\n")))
+	}
+}
+
+// NewStatefulSetPVC returns a PersistentVolumeClaim named name, for testing StatefulSets.
+func NewStatefulSetPVC(name string) v1.PersistentVolumeClaim {
+	return v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: *resource.NewQuantity(1, resource.BinarySI),
+				},
+			},
+		},
+	}
+}
+
+// NewStatefulSet creates a new NGINX StatefulSet for testing. The StatefulSet is named name, is in namespace ns,
+// statefulPodsMounts are the mounts that will be backed by PVs. podsMounts are the mounts that are mounted directly
+// to the Pod. labels are the labels that will be usd for the StatefulSet selector.
+func NewStatefulSet(name, ns, governingSvcName string, replicas int32, statefulPodMounts []v1.VolumeMount, podMounts []v1.VolumeMount, labels map[string]string) *apps.StatefulSet {
+	mounts := append(statefulPodMounts, podMounts...)
+	claims := []v1.PersistentVolumeClaim{}
+	for _, m := range statefulPodMounts {
+		claims = append(claims, NewStatefulSetPVC(m.Name))
+	}
+
+	vols := []v1.Volume{}
+	for _, m := range podMounts {
+		vols = append(vols, v1.Volume{
+			Name: m.Name,
+			VolumeSource: v1.VolumeSource{
+				HostPath: &v1.HostPathVolumeSource{
+					Path: fmt.Sprintf("/tmp/%v", m.Name),
+				},
+			},
+		})
+	}
+
+	return &apps.StatefulSet{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "StatefulSet",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: ns,
+		},
+		Spec: apps.StatefulSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: labels,
+			},
+			Replicas: func(i int32) *int32 { return &i }(replicas),
+			Template: v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels:      labels,
+					Annotations: map[string]string{},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:         "nginx",
+							Image:        imageutils.GetE2EImage(imageutils.Nginx),
+							VolumeMounts: mounts,
+						},
+					},
+					Volumes: vols,
+				},
+			},
+			UpdateStrategy:       apps.StatefulSetUpdateStrategy{Type: apps.RollingUpdateStatefulSetStrategyType},
+			VolumeClaimTemplates: claims,
+			ServiceName:          governingSvcName,
+		},
+	}
+}
+
+// NewStatefulSetScale creates a new StatefulSet scale subresource and returns it
+func NewStatefulSetScale(ss *apps.StatefulSet) *appsV1beta2.Scale {
+	return &appsV1beta2.Scale{
+		// TODO: Create a variant of ObjectMeta type that only contains the fields below.
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ss.Name,
+			Namespace: ss.Namespace,
+		},
+		Spec: appsV1beta2.ScaleSpec{
+			Replicas: *(ss.Spec.Replicas),
+		},
+		Status: appsV1beta2.ScaleStatus{
+			Replicas: ss.Status.Replicas,
+		},
+	}
+}
+
+var statefulPodRegex = regexp.MustCompile("(.*)-([0-9]+)$")
+
+func getStatefulPodOrdinal(pod *v1.Pod) int {
+	ordinal := -1
+	subMatches := statefulPodRegex.FindStringSubmatch(pod.Name)
+	if len(subMatches) < 3 {
+		return ordinal
+	}
+	if i, err := strconv.ParseInt(subMatches[2], 10, 32); err == nil {
+		ordinal = int(i)
+	}
+	return ordinal
+}
+
+type statefulPodsByOrdinal []v1.Pod
+
+func (sp statefulPodsByOrdinal) Len() int {
+	return len(sp)
+}
+
+func (sp statefulPodsByOrdinal) Swap(i, j int) {
+	sp[i], sp[j] = sp[j], sp[i]
+}
+
+func (sp statefulPodsByOrdinal) Less(i, j int) bool {
+	return getStatefulPodOrdinal(&sp[i]) < getStatefulPodOrdinal(&sp[j])
+}
+
+type updateStatefulSetFunc func(*apps.StatefulSet)
+
+func UpdateStatefulSetWithRetries(c clientset.Interface, namespace, name string, applyUpdate updateStatefulSetFunc) (statefulSet *apps.StatefulSet, err error) {
+	statefulSets := c.AppsV1().StatefulSets(namespace)
+	var updateErr error
+	pollErr := wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
+		if statefulSet, err = statefulSets.Get(name, metav1.GetOptions{}); err != nil {
+			return false, err
+		}
+		// Apply the update, then attempt to push it to the apiserver.
+		applyUpdate(statefulSet)
+		if statefulSet, err = statefulSets.Update(statefulSet); err == nil {
+			Logf("Updating stateful set %s", name)
+			return true, nil
+		}
+		updateErr = err
+		return false, nil
+	})
+	if pollErr == wait.ErrWaitTimeout {
+		pollErr = fmt.Errorf("couldn't apply the provided updated to stateful set %q: %v", name, updateErr)
+	}
+	return statefulSet, pollErr
+}

vendor/k8s.io/kubernetes/test/e2e/framework/test_context.go

@@ -0,0 +1,448 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/onsi/ginkgo/config"
+	"github.com/pkg/errors"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
+)
+
+const (
+	defaultHost = "http://127.0.0.1:8080"
+
+	// DefaultNumNodes is the number of nodes. If not specified, then number of nodes is auto-detected
+	DefaultNumNodes = -1
+)
+
+// TestContextType contains test settings and global state. Due to
+// historic reasons, it is a mixture of items managed by the test
+// framework itself, cloud providers and individual tests.
+// The goal is to move anything not required by the framework
+// into the code which uses the settings.
+//
+// The recommendation for those settings is:
+// - They are stored in their own context structure or local
+//   variables.
+// - The standard `flag` package is used to register them.
+//   The flag name should follow the pattern <part1>.<part2>....<partn>
+//   where the prefix is unlikely to conflict with other tests or
+//   standard packages and each part is in lower camel case. For
+//   example, test/e2e/storage/csi/context.go could define
+//   storage.csi.numIterations.
+// - framework/config can be used to simplify the registration of
+//   multiple options with a single function call:
+//   var storageCSI {
+//       NumIterations `default:"1" usage:"number of iterations"`
+//   }
+//   _ config.AddOptions(&storageCSI, "storage.csi")
+// - The direct use Viper in tests is possible, but discouraged because
+//   it only works in test suites which use Viper (which is not
+//   required) and the supported options cannot be
+//   discovered by a test suite user.
+//
+// Test suite authors can use framework/viper to make all command line
+// parameters also configurable via a configuration file.
+type TestContextType struct {
+	KubeConfig         string
+	KubeContext        string
+	KubeAPIContentType string
+	KubeVolumeDir      string
+	CertDir            string
+	Host               string
+	// TODO: Deprecating this over time... instead just use gobindata_util.go , see #23987.
+	RepoRoot                string
+	DockershimCheckpointDir string
+	// ListImages will list off all images that are used then quit
+	ListImages bool
+
+	// Provider identifies the infrastructure provider (gce, gke, aws)
+	Provider string
+
+	// Tooling is the tooling in use (e.g. kops, gke).  Provider is the cloud provider and might not uniquely identify the tooling.
+	Tooling string
+
+	CloudConfig    CloudConfig
+	KubectlPath    string
+	OutputDir      string
+	ReportDir      string
+	ReportPrefix   string
+	Prefix         string
+	MinStartupPods int
+	// Timeout for waiting for system pods to be running
+	SystemPodsStartupTimeout    time.Duration
+	EtcdUpgradeStorage          string
+	EtcdUpgradeVersion          string
+	IngressUpgradeImage         string
+	GCEUpgradeScript            string
+	ContainerRuntime            string
+	ContainerRuntimeEndpoint    string
+	ContainerRuntimeProcessName string
+	ContainerRuntimePidFile     string
+	// SystemdServices are comma separated list of systemd services the test framework
+	// will dump logs for.
+	SystemdServices string
+	// DumpSystemdJournal controls whether to dump the full systemd journal.
+	DumpSystemdJournal       bool
+	ImageServiceEndpoint     string
+	MasterOSDistro           string
+	NodeOSDistro             string
+	VerifyServiceAccount     bool
+	DeleteNamespace          bool
+	DeleteNamespaceOnFailure bool
+	AllowedNotReadyNodes     int
+	CleanStart               bool
+	// If set to 'true' or 'all' framework will start a goroutine monitoring resource usage of system add-ons.
+	// It will read the data every 30 seconds from all Nodes and print summary during afterEach. If set to 'master'
+	// only master Node will be monitored.
+	GatherKubeSystemResourceUsageData string
+	GatherLogsSizes                   bool
+	GatherMetricsAfterTest            string
+	GatherSuiteMetricsAfterTest       bool
+	MaxNodesToGather                  int
+	AllowGatheringProfiles            bool
+	// If set to 'true' framework will gather ClusterAutoscaler metrics when gathering them for other components.
+	IncludeClusterAutoscalerMetrics bool
+	// Currently supported values are 'hr' for human-readable and 'json'. It's a comma separated list.
+	OutputPrintType string
+	// NodeSchedulableTimeout is the timeout for waiting for all nodes to be schedulable.
+	NodeSchedulableTimeout time.Duration
+	// SystemDaemonsetStartupTimeout is the timeout for waiting for all system daemonsets to be ready.
+	SystemDaemonsetStartupTimeout time.Duration
+	// CreateTestingNS is responsible for creating namespace used for executing e2e tests.
+	// It accepts namespace base name, which will be prepended with e2e prefix, kube client
+	// and labels to be applied to a namespace.
+	CreateTestingNS CreateTestingNSFn
+	// If set to true test will dump data about the namespace in which test was running.
+	DumpLogsOnFailure bool
+	// Disables dumping cluster log from master and nodes after all tests.
+	DisableLogDump bool
+	// Path to the GCS artifacts directory to dump logs from nodes. Logexporter gets enabled if this is non-empty.
+	LogexporterGCSPath string
+	// featureGates is a map of feature names to bools that enable or disable alpha/experimental features.
+	FeatureGates map[string]bool
+	// Node e2e specific test context
+	NodeTestContextType
+	// Monitoring solution that is used in current cluster.
+	ClusterMonitoringMode string
+	// Separate Prometheus monitoring deployed in cluster
+	EnablePrometheusMonitoring bool
+
+	// Indicates what path the kubernetes-anywhere is installed on
+	KubernetesAnywherePath string
+
+	// The DNS Domain of the cluster.
+	ClusterDNSDomain string
+
+	// The configration of NodeKiller.
+	NodeKiller NodeKillerConfig
+}
+
+// NodeKillerConfig describes configuration of NodeKiller -- a utility to
+// simulate node failures.
+type NodeKillerConfig struct {
+	// Enabled determines whether NodeKill should do anything at all.
+	// All other options below are ignored if Enabled = false.
+	Enabled bool
+	// FailureRatio is a percentage of all nodes that could fail simultinously.
+	FailureRatio float64
+	// Interval is time between node failures.
+	Interval time.Duration
+	// JitterFactor is factor used to jitter node failures.
+	// Node will be killed between [Interval, Interval + (1.0 + JitterFactor)].
+	JitterFactor float64
+	// SimulatedDowntime is a duration between node is killed and recreated.
+	SimulatedDowntime time.Duration
+}
+
+// NodeTestContextType is part of TestContextType, it is shared by all node e2e test.
+type NodeTestContextType struct {
+	// NodeE2E indicates whether it is running node e2e.
+	NodeE2E bool
+	// Name of the node to run tests on.
+	NodeName string
+	// NodeConformance indicates whether the test is running in node conformance mode.
+	NodeConformance bool
+	// PrepullImages indicates whether node e2e framework should prepull images.
+	PrepullImages bool
+	// KubeletConfig is the kubelet configuration the test is running against.
+	KubeletConfig kubeletconfig.KubeletConfiguration
+	// ImageDescription is the description of the image on which the test is running.
+	ImageDescription string
+	// SystemSpecName is the name of the system spec (e.g., gke) that's used in
+	// the node e2e test. If empty, the default one (system.DefaultSpec) is
+	// used. The system specs are in test/e2e_node/system/specs/.
+	SystemSpecName string
+	// ExtraEnvs is a map of environment names to values.
+	ExtraEnvs map[string]string
+}
+
+type CloudConfig struct {
+	ApiEndpoint       string
+	ProjectID         string
+	Zone              string // for multizone tests, arbitrarily chosen zone
+	Region            string
+	MultiZone         bool
+	MultiMaster       bool
+	Cluster           string
+	MasterName        string
+	NodeInstanceGroup string // comma-delimited list of groups' names
+	NumNodes          int
+	ClusterIPRange    string
+	ClusterTag        string
+	Network           string
+	ConfigFile        string // for azure and openstack
+	NodeTag           string
+	MasterTag         string
+
+	Provider ProviderInterface
+}
+
+var TestContext TestContextType
+
+// Register flags common to all e2e test suites.
+func RegisterCommonFlags() {
+	// Turn on verbose by default to get spec names
+	config.DefaultReporterConfig.Verbose = true
+
+	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
+	config.GinkgoConfig.EmitSpecProgress = true
+
+	// Randomize specs as well as suites
+	config.GinkgoConfig.RandomizeAllSpecs = true
+
+	flag.StringVar(&TestContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", "false", "If set to 'true' or 'all' framework will be monitoring resource usage of system all add-ons in (some) e2e tests, if set to 'master' framework will be monitoring master node only, if set to 'none' of 'false' monitoring will be turned off.")
+	flag.BoolVar(&TestContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
+	flag.IntVar(&TestContext.MaxNodesToGather, "max-nodes-to-gather-from", 20, "The maximum number of nodes to gather extended info from on test failure.")
+	flag.StringVar(&TestContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", "false", "If set to 'true' framework will gather metrics from all components after each test. If set to 'master' only master component metrics would be gathered.")
+	flag.BoolVar(&TestContext.GatherSuiteMetricsAfterTest, "gather-suite-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after the whole test suite completes.")
+	flag.BoolVar(&TestContext.AllowGatheringProfiles, "allow-gathering-profiles", true, "If set to true framework will allow to gather CPU/memory allocation pprof profiles from the master.")
+	flag.BoolVar(&TestContext.IncludeClusterAutoscalerMetrics, "include-cluster-autoscaler", false, "If set to true, framework will include Cluster Autoscaler when gathering metrics.")
+	flag.StringVar(&TestContext.OutputPrintType, "output-print-type", "json", "Format in which summaries should be printed: 'hr' for human readable, 'json' for JSON ones.")
+	flag.BoolVar(&TestContext.DumpLogsOnFailure, "dump-logs-on-failure", true, "If set to true test will dump data about the namespace in which test was running.")
+	flag.BoolVar(&TestContext.DisableLogDump, "disable-log-dump", false, "If set to true, logs from master and nodes won't be gathered after test run.")
+	flag.StringVar(&TestContext.LogexporterGCSPath, "logexporter-gcs-path", "", "Path to the GCS artifacts directory to dump logs from nodes. Logexporter gets enabled if this is non-empty.")
+	flag.BoolVar(&TestContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
+	flag.BoolVar(&TestContext.DeleteNamespaceOnFailure, "delete-namespace-on-failure", true, "If true, framework will delete test namespace on failure. Used only during test debugging.")
+	flag.IntVar(&TestContext.AllowedNotReadyNodes, "allowed-not-ready-nodes", 0, "If non-zero, framework will allow for that many non-ready nodes when checking for all ready nodes.")
+
+	flag.StringVar(&TestContext.Host, "host", "", fmt.Sprintf("The host, or apiserver, to connect to. Will default to %s if this argument and --kubeconfig are not set", defaultHost))
+	flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
+	flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
+	flag.Var(cliflag.NewMapStringBool(&TestContext.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
+	flag.StringVar(&TestContext.ContainerRuntime, "container-runtime", "docker", "The container runtime of cluster VM instances (docker/remote).")
+	flag.StringVar(&TestContext.ContainerRuntimeEndpoint, "container-runtime-endpoint", "unix:///var/run/dockershim.sock", "The container runtime endpoint of cluster VM instances.")
+	flag.StringVar(&TestContext.ContainerRuntimeProcessName, "container-runtime-process-name", "dockerd", "The name of the container runtime process.")
+	flag.StringVar(&TestContext.ContainerRuntimePidFile, "container-runtime-pid-file", "/var/run/docker.pid", "The pid file of the container runtime.")
+	flag.StringVar(&TestContext.SystemdServices, "systemd-services", "docker", "The comma separated list of systemd services the framework will dump logs for.")
+	flag.BoolVar(&TestContext.DumpSystemdJournal, "dump-systemd-journal", false, "Whether to dump the full systemd journal.")
+	flag.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.")
+	flag.StringVar(&TestContext.DockershimCheckpointDir, "dockershim-checkpoint-dir", "/var/lib/dockershim/sandbox", "The directory for dockershim to store sandbox checkpoints.")
+	flag.StringVar(&TestContext.KubernetesAnywherePath, "kubernetes-anywhere-path", "/workspace/k8s.io/kubernetes-anywhere", "Which directory kubernetes-anywhere is installed to.")
+
+	flag.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.")
+}
+
+// Register flags specific to the cluster e2e test suite.
+func RegisterClusterFlags() {
+	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
+	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
+	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
+	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
+
+	flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
+	flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
+	flag.StringVar(&TestContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.")
+	flag.StringVar(&TestContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, skeleton (the fallback if not set), etc.)")
+	flag.StringVar(&TestContext.Tooling, "tooling", "", "The tooling in use (kops, gke, etc.)")
+	flag.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
+	flag.StringVar(&TestContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")
+	flag.StringVar(&TestContext.Prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.")
+	flag.StringVar(&TestContext.MasterOSDistro, "master-os-distro", "debian", "The OS distribution of cluster master (debian, ubuntu, gci, coreos, or custom).")
+	flag.StringVar(&TestContext.NodeOSDistro, "node-os-distro", "debian", "The OS distribution of cluster VM instances (debian, ubuntu, gci, coreos, or custom).")
+	flag.StringVar(&TestContext.ClusterMonitoringMode, "cluster-monitoring-mode", "standalone", "The monitoring solution that is used in the cluster.")
+	flag.BoolVar(&TestContext.EnablePrometheusMonitoring, "prometheus-monitoring", false, "Separate Prometheus monitoring deployed in cluster.")
+	flag.StringVar(&TestContext.ClusterDNSDomain, "dns-domain", "cluster.local", "The DNS Domain of the cluster.")
+
+	// TODO: Flags per provider?  Rename gce-project/gce-zone?
+	cloudConfig := &TestContext.CloudConfig
+	flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke")
+	flag.StringVar(&cloudConfig.ApiEndpoint, "gce-api-endpoint", "", "The GCE APIEndpoint being used, if applicable")
+	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
+	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
+	flag.StringVar(&cloudConfig.Region, "gce-region", "", "GCE region being used, if applicable")
+	flag.BoolVar(&cloudConfig.MultiZone, "gce-multizone", false, "If true, start GCE cloud provider with multizone support.")
+	flag.BoolVar(&cloudConfig.MultiMaster, "gce-multimaster", false, "If true, the underlying GCE/GKE cluster is assumed to be multi-master.")
+	flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable")
+	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws. If there is more than one group: comma separated list of groups.")
+	flag.StringVar(&cloudConfig.Network, "network", "e2e", "The cloud provider network for this e2e cluster.")
+	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", DefaultNumNodes, fmt.Sprintf("Number of nodes in the cluster. If the default value of '%q' is used the number of schedulable nodes is auto-detected.", DefaultNumNodes))
+	flag.StringVar(&cloudConfig.ClusterIPRange, "cluster-ip-range", "10.64.0.0/14", "A CIDR notation IP range from which to assign IPs in the cluster.")
+	flag.StringVar(&cloudConfig.NodeTag, "node-tag", "", "Network tags used on node instances. Valid only for gce, gke")
+	flag.StringVar(&cloudConfig.MasterTag, "master-tag", "", "Network tags used on master instances. Valid only for gce, gke")
+
+	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
+	flag.StringVar(&cloudConfig.ConfigFile, "cloud-config-file", "", "Cloud config file.  Only required if provider is azure.")
+	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
+	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
+	flag.DurationVar(&TestContext.NodeSchedulableTimeout, "node-schedulable-timeout", 30*time.Minute, "Timeout for waiting for all nodes to be schedulable.")
+	flag.DurationVar(&TestContext.SystemDaemonsetStartupTimeout, "system-daemonsets-startup-timeout", 5*time.Minute, "Timeout for waiting for all system daemonsets to be ready.")
+	flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.")
+	flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.")
+	flag.StringVar(&TestContext.IngressUpgradeImage, "ingress-upgrade-image", "", "Image to upgrade to if doing an upgrade test for ingress.")
+	flag.StringVar(&TestContext.GCEUpgradeScript, "gce-upgrade-script", "", "Script to use to upgrade a GCE cluster.")
+	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
+
+	nodeKiller := &TestContext.NodeKiller
+	flag.BoolVar(&nodeKiller.Enabled, "node-killer", false, "Whether NodeKiller should kill any nodes.")
+	flag.Float64Var(&nodeKiller.FailureRatio, "node-killer-failure-ratio", 0.01, "Percentage of nodes to be killed")
+	flag.DurationVar(&nodeKiller.Interval, "node-killer-interval", 1*time.Minute, "Time between node failures.")
+	flag.Float64Var(&nodeKiller.JitterFactor, "node-killer-jitter-factor", 60, "Factor used to jitter node failures.")
+	flag.DurationVar(&nodeKiller.SimulatedDowntime, "node-killer-simulated-downtime", 10*time.Minute, "A delay between node death and recreation")
+}
+
+// Register flags specific to the node e2e test suite.
+func RegisterNodeFlags() {
+	// Mark the test as node e2e when node flags are api.Registry.
+	TestContext.NodeE2E = true
+	flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on.")
+	// TODO(random-liu): Move kubelet start logic out of the test.
+	// TODO(random-liu): Move log fetch logic out of the test.
+	// There are different ways to start kubelet (systemd, initd, docker, manually started etc.)
+	// and manage logs (journald, upstart etc.).
+	// For different situation we need to mount different things into the container, run different commands.
+	// It is hard and unnecessary to deal with the complexity inside the test suite.
+	flag.BoolVar(&TestContext.NodeConformance, "conformance", false, "If true, the test suite will not start kubelet, and fetch system log (kernel, docker, kubelet log etc.) to the report directory.")
+	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+	flag.StringVar(&TestContext.ImageDescription, "image-description", "", "The description of the image which the test will be running on.")
+	flag.StringVar(&TestContext.SystemSpecName, "system-spec-name", "", "The name of the system spec (e.g., gke) that's used in the node e2e test. The system specs are in test/e2e_node/system/specs/. This is used by the test framework to determine which tests to run for validating the system requirements.")
+	flag.Var(cliflag.NewMapStringString(&TestContext.ExtraEnvs), "extra-envs", "The extra environment variables needed for node e2e tests. Format: a list of key=value pairs, e.g., env1=val1,env2=val2")
+}
+
+// HandleFlags sets up all flags and parses the command line.
+func HandleFlags() {
+	RegisterCommonFlags()
+	RegisterClusterFlags()
+	flag.Parse()
+}
+
+func createKubeConfig(clientCfg *restclient.Config) *clientcmdapi.Config {
+	clusterNick := "cluster"
+	userNick := "user"
+	contextNick := "context"
+
+	config := clientcmdapi.NewConfig()
+
+	credentials := clientcmdapi.NewAuthInfo()
+	credentials.Token = clientCfg.BearerToken
+	credentials.TokenFile = clientCfg.BearerTokenFile
+	credentials.ClientCertificate = clientCfg.TLSClientConfig.CertFile
+	if len(credentials.ClientCertificate) == 0 {
+		credentials.ClientCertificateData = clientCfg.TLSClientConfig.CertData
+	}
+	credentials.ClientKey = clientCfg.TLSClientConfig.KeyFile
+	if len(credentials.ClientKey) == 0 {
+		credentials.ClientKeyData = clientCfg.TLSClientConfig.KeyData
+	}
+	config.AuthInfos[userNick] = credentials
+
+	cluster := clientcmdapi.NewCluster()
+	cluster.Server = clientCfg.Host
+	cluster.CertificateAuthority = clientCfg.CAFile
+	if len(cluster.CertificateAuthority) == 0 {
+		cluster.CertificateAuthorityData = clientCfg.CAData
+	}
+	cluster.InsecureSkipTLSVerify = clientCfg.Insecure
+	config.Clusters[clusterNick] = cluster
+
+	context := clientcmdapi.NewContext()
+	context.Cluster = clusterNick
+	context.AuthInfo = userNick
+	config.Contexts[contextNick] = context
+	config.CurrentContext = contextNick
+
+	return config
+}
+
+// AfterReadingAllFlags makes changes to the context after all flags
+// have been read.
+func AfterReadingAllFlags(t *TestContextType) {
+	// Only set a default host if one won't be supplied via kubeconfig
+	if len(t.Host) == 0 && len(t.KubeConfig) == 0 {
+		// Check if we can use the in-cluster config
+		if clusterConfig, err := restclient.InClusterConfig(); err == nil {
+			if tempFile, err := ioutil.TempFile(os.TempDir(), "kubeconfig-"); err == nil {
+				kubeConfig := createKubeConfig(clusterConfig)
+				clientcmd.WriteToFile(*kubeConfig, tempFile.Name())
+				t.KubeConfig = tempFile.Name()
+				klog.Infof("Using a temporary kubeconfig file from in-cluster config : %s", tempFile.Name())
+			}
+		}
+		if len(t.KubeConfig) == 0 {
+			klog.Warningf("Unable to find in-cluster config, using default host : %s", defaultHost)
+			t.Host = defaultHost
+		}
+	}
+	// Allow 1% of nodes to be unready (statistically) - relevant for large clusters.
+	if t.AllowedNotReadyNodes == 0 {
+		t.AllowedNotReadyNodes = t.CloudConfig.NumNodes / 100
+	}
+
+	// Make sure that all test runs have a valid TestContext.CloudConfig.Provider.
+	// TODO: whether and how long this code is needed is getting discussed
+	// in https://github.com/kubernetes/kubernetes/issues/70194.
+	if TestContext.Provider == "" {
+		// Some users of the e2e.test binary pass --provider=.
+		// We need to support that, changing it would break those usages.
+		Logf("The --provider flag is not set. Continuing as if --provider=skeleton had been used.")
+		TestContext.Provider = "skeleton"
+	}
+
+	var err error
+	TestContext.CloudConfig.Provider, err = SetupProviderConfig(TestContext.Provider)
+	if err != nil {
+		if os.IsNotExist(errors.Cause(err)) {
+			// Provide a more helpful error message when the provider is unknown.
+			var providers []string
+			for _, name := range GetProviders() {
+				// The empty string is accepted, but looks odd in the output below unless we quote it.
+				if name == "" {
+					name = `""`
+				}
+				providers = append(providers, name)
+			}
+			sort.Strings(providers)
+			klog.Errorf("Unknown provider %q. The following providers are known: %v", TestContext.Provider, strings.Join(providers, " "))
+		} else {
+			klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err)
+		}
+		os.Exit(1)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/testfiles/testfiles.go

@@ -0,0 +1,196 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package testfiles provides a wrapper around various optional ways
+// of retrieving additional files needed during a test run:
+// - builtin bindata
+// - filesystem access
+//
+// Because it is a is self-contained package, it can be used by
+// test/e2e/framework and test/e2e/manifest without creating
+// a circular dependency.
+package testfiles
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+var filesources []FileSource
+
+// AddFileSource registers another provider for files that may be
+// needed at runtime. Should be called during initialization of a test
+// binary.
+func AddFileSource(filesource FileSource) {
+	filesources = append(filesources, filesource)
+}
+
+// FileSource implements one way of retrieving test file content.  For
+// example, one file source could read from the original source code
+// file tree, another from bindata compiled into a test executable.
+type FileSource interface {
+	// ReadTestFile retrieves the content of a file that gets maintained
+	// alongside a test's source code. Files are identified by the
+	// relative path inside the repository containing the tests, for
+	// example "cluster/gce/upgrade.sh" inside kubernetes/kubernetes.
+	//
+	// When the file is not found, a nil slice is returned. An error is
+	// returned for all fatal errors.
+	ReadTestFile(filePath string) ([]byte, error)
+
+	// DescribeFiles returns a multi-line description of which
+	// files are available via this source. It is meant to be
+	// used as part of the error message when a file cannot be
+	// found.
+	DescribeFiles() string
+}
+
+// Fail is an error handler function with the same prototype and
+// semantic as ginkgo.Fail. Typically ginkgo.Fail is what callers
+// of ReadOrDie and Exists will pass. This way this package
+// avoids depending on Ginkgo.
+type Fail func(failure string, callerSkip ...int)
+
+// ReadOrDie tries to retrieve the desired file content from
+// one of the registered file sources. In contrast to FileSource, it
+// will either return a valid slice or abort the test by calling the fatal function,
+// i.e. the caller doesn't have to implement error checking.
+func ReadOrDie(filePath string, fail Fail) []byte {
+	data, err := Read(filePath)
+	if err != nil {
+		fail(err.Error(), 1)
+	}
+	return data
+}
+
+// Read tries to retrieve the desired file content from
+// one of the registered file sources.
+func Read(filePath string) ([]byte, error) {
+	if len(filesources) == 0 {
+		return nil, fmt.Errorf("no file sources registered (yet?), cannot retrieve test file %s", filePath)
+	}
+	for _, filesource := range filesources {
+		data, err := filesource.ReadTestFile(filePath)
+		if err != nil {
+			return nil, fmt.Errorf("fatal error retrieving test file %s: %s", filePath, err)
+		}
+		if data != nil {
+			return data, nil
+		}
+	}
+	// Here we try to generate an error that points test authors
+	// or users in the right direction for resolving the problem.
+	error := fmt.Sprintf("Test file %q was not found.\n", filePath)
+	for _, filesource := range filesources {
+		error += filesource.DescribeFiles()
+		error += "\n"
+	}
+	return nil, errors.New(error)
+}
+
+// Exists checks whether a file could be read. Unexpected errors
+// are handled by calling the fail function, which then should
+// abort the current test.
+func Exists(filePath string, fail Fail) bool {
+	for _, filesource := range filesources {
+		data, err := filesource.ReadTestFile(filePath)
+		if err != nil {
+			fail(fmt.Sprintf("fatal error looking for test file %s: %s", filePath, err), 1)
+		}
+		if data != nil {
+			return true
+		}
+	}
+	return false
+}
+
+// RootFileSource looks for files relative to a root directory.
+type RootFileSource struct {
+	Root string
+}
+
+// ReadTestFile looks for the file relative to the configured
+// root directory. If the path is already absolute, for example
+// in a test that has its own method of determining where
+// files are, then the path will be used directly.
+func (r RootFileSource) ReadTestFile(filePath string) ([]byte, error) {
+	var fullPath string
+	if path.IsAbs(filePath) {
+		fullPath = filePath
+	} else {
+		fullPath = filepath.Join(r.Root, filePath)
+	}
+	data, err := ioutil.ReadFile(fullPath)
+	if os.IsNotExist(err) {
+		// Not an error (yet), some other provider may have the file.
+		return nil, nil
+	}
+	return data, err
+}
+
+// DescribeFiles explains that it looks for files inside a certain
+// root directory.
+func (r RootFileSource) DescribeFiles() string {
+	description := fmt.Sprintf("Test files are expected in %q", r.Root)
+	if !path.IsAbs(r.Root) {
+		// The default in test_context.go is the relative path
+		// ../../, which doesn't really help locating the
+		// actual location. Therefore we add also the absolute
+		// path if necessary.
+		abs, err := filepath.Abs(r.Root)
+		if err == nil {
+			description += fmt.Sprintf(" = %q", abs)
+		}
+	}
+	description += "."
+	return description
+}
+
+// BindataFileSource handles files stored in a package generated with bindata.
+type BindataFileSource struct {
+	Asset      func(string) ([]byte, error)
+	AssetNames func() []string
+}
+
+// ReadTestFile looks for an asset with the given path.
+func (b BindataFileSource) ReadTestFile(filePath string) ([]byte, error) {
+	fileBytes, err := b.Asset(filePath)
+	if err != nil {
+		// It would be nice to have a better way to detect
+		// "not found" errors :-/
+		if strings.HasSuffix(err.Error(), "not found") {
+			return nil, nil
+		}
+	}
+	return fileBytes, nil
+}
+
+// DescribeFiles explains about gobindata and then lists all available files.
+func (b BindataFileSource) DescribeFiles() string {
+	var lines []string
+	lines = append(lines, "The following files are built into the test executable via gobindata. For questions on maintaining gobindata, contact the sig-testing group.")
+	assets := b.AssetNames()
+	sort.Strings(assets)
+	lines = append(lines, assets...)
+	description := strings.Join(lines, "\n   ")
+	return description
+}

vendor/k8s.io/kubernetes/test/e2e/framework/upgrade_util.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"os/exec"
+	"path"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+// RealVersion turns a version constants into a version string deployable on
+// GKE.  See hack/get-build.sh for more information.
+func RealVersion(s string) (string, error) {
+	Logf("Getting real version for %q", s)
+	v, _, err := RunCmd(path.Join(TestContext.RepoRoot, "hack/get-build.sh"), "-v", s)
+	if err != nil {
+		return v, err
+	}
+	Logf("Version for %q is %q", s, v)
+	return strings.TrimPrefix(strings.TrimSpace(v), "v"), nil
+}
+
+func traceRouteToMaster() {
+	path, err := exec.LookPath("traceroute")
+	if err != nil {
+		Logf("Could not find traceroute program")
+		return
+	}
+
+	cmd := exec.Command(path, "-I", GetMasterHost())
+	out, err := cmd.Output()
+	if len(out) != 0 {
+		Logf(string(out))
+	}
+	if exiterr, ok := err.(*exec.ExitError); err != nil && ok {
+		Logf("error while running traceroute: %s", exiterr.Stderr)
+	}
+}
+
+func CheckMasterVersion(c clientset.Interface, want string) error {
+	Logf("Checking master version")
+	var err error
+	var v *version.Info
+	waitErr := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
+		v, err = c.Discovery().ServerVersion()
+		if err != nil {
+			traceRouteToMaster()
+			return false, nil
+		}
+		return true, nil
+	})
+	if waitErr != nil {
+		return fmt.Errorf("CheckMasterVersion() couldn't get the master version: %v", err)
+	}
+	// We do prefix trimming and then matching because:
+	// want looks like:  0.19.3-815-g50e67d4
+	// got  looks like: v0.19.3-815-g50e67d4034e858-dirty
+	got := strings.TrimPrefix(v.GitVersion, "v")
+	if !strings.HasPrefix(got, want) {
+		return fmt.Errorf("master had kube-apiserver version %s which does not start with %s",
+			got, want)
+	}
+	Logf("Master is at version %s", want)
+	return nil
+}
+
+func CheckNodesVersions(cs clientset.Interface, want string) error {
+	l := GetReadySchedulableNodesOrDie(cs)
+	for _, n := range l.Items {
+		// We do prefix trimming and then matching because:
+		// want   looks like:  0.19.3-815-g50e67d4
+		// kv/kvp look  like: v0.19.3-815-g50e67d4034e858-dirty
+		kv, kpv := strings.TrimPrefix(n.Status.NodeInfo.KubeletVersion, "v"),
+			strings.TrimPrefix(n.Status.NodeInfo.KubeProxyVersion, "v")
+		if !strings.HasPrefix(kv, want) {
+			return fmt.Errorf("node %s had kubelet version %s which does not start with %s",
+				n.ObjectMeta.Name, kv, want)
+		}
+		if !strings.HasPrefix(kpv, want) {
+			return fmt.Errorf("node %s had kube-proxy version %s which does not start with %s",
+				n.ObjectMeta.Name, kpv, want)
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/volume_util.go

@@ -0,0 +1,651 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ * This test checks that various VolumeSources are working.
+ *
+ * There are two ways, how to test the volumes:
+ * 1) With containerized server (NFS, Ceph, Gluster, iSCSI, ...)
+ * The test creates a server pod, exporting simple 'index.html' file.
+ * Then it uses appropriate VolumeSource to import this file into a client pod
+ * and checks that the pod can see the file. It does so by importing the file
+ * into web server root and loadind the index.html from it.
+ *
+ * These tests work only when privileged containers are allowed, exporting
+ * various filesystems (NFS, GlusterFS, ...) usually needs some mounting or
+ * other privileged magic in the server pod.
+ *
+ * Note that the server containers are for testing purposes only and should not
+ * be used in production.
+ *
+ * 2) With server outside of Kubernetes (Cinder, ...)
+ * Appropriate server (e.g. OpenStack Cinder) must exist somewhere outside
+ * the tested Kubernetes cluster. The test itself creates a new volume,
+ * and checks, that Kubernetes can use it as a volume.
+ */
+
+package framework
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"strconv"
+	"time"
+
+	"k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+	clientset "k8s.io/client-go/kubernetes"
+	imageutils "k8s.io/kubernetes/test/utils/image"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	Kb  int64 = 1000
+	Mb  int64 = 1000 * Kb
+	Gb  int64 = 1000 * Mb
+	Tb  int64 = 1000 * Gb
+	KiB int64 = 1024
+	MiB int64 = 1024 * KiB
+	GiB int64 = 1024 * MiB
+	TiB int64 = 1024 * GiB
+
+	// Waiting period for volume server (Ceph, ...) to initialize itself.
+	VolumeServerPodStartupTimeout = 3 * time.Minute
+
+	// Waiting period for pod to be cleaned up and unmount its volumes so we
+	// don't tear down containers with NFS/Ceph/Gluster server too early.
+	PodCleanupTimeout = 20 * time.Second
+)
+
+// Configuration of one tests. The test consist of:
+// - server pod - runs serverImage, exports ports[]
+// - client pod - does not need any special configuration
+type VolumeTestConfig struct {
+	Namespace string
+	// Prefix of all pods. Typically the test name.
+	Prefix string
+	// Name of container image for the server pod.
+	ServerImage string
+	// Ports to export from the server pod. TCP only.
+	ServerPorts []int
+	// Commands to run in the container image.
+	ServerCmds []string
+	// Arguments to pass to the container image.
+	ServerArgs []string
+	// Volumes needed to be mounted to the server container from the host
+	// map <host (source) path> -> <container (dst.) path>
+	// if <host (source) path> is empty, mount a tmpfs emptydir
+	ServerVolumes map[string]string
+	// Message to wait for before starting clients
+	ServerReadyMessage string
+	// Wait for the pod to terminate successfully
+	// False indicates that the pod is long running
+	WaitForCompletion bool
+	// ServerNodeName is the spec.nodeName to run server pod on.  Default is any node.
+	ServerNodeName string
+	// ClientNodeName is the spec.nodeName to run client pod on.  Default is any node.
+	ClientNodeName string
+	// NodeSelector to use in pod spec (server, client and injector pods).
+	NodeSelector map[string]string
+}
+
+// VolumeTest contains a volume to mount into a client pod and its
+// expected content.
+type VolumeTest struct {
+	Volume          v1.VolumeSource
+	File            string
+	ExpectedContent string
+}
+
+// NFS-specific wrapper for CreateStorageServer.
+func NewNFSServer(cs clientset.Interface, namespace string, args []string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
+	config = VolumeTestConfig{
+		Namespace:          namespace,
+		Prefix:             "nfs",
+		ServerImage:        imageutils.GetE2EImage(imageutils.VolumeNFSServer),
+		ServerPorts:        []int{2049},
+		ServerVolumes:      map[string]string{"": "/exports"},
+		ServerReadyMessage: "NFS started",
+	}
+	if len(args) > 0 {
+		config.ServerArgs = args
+	}
+	pod, ip = CreateStorageServer(cs, config)
+	return config, pod, ip
+}
+
+// GlusterFS-specific wrapper for CreateStorageServer. Also creates the gluster endpoints object.
+func NewGlusterfsServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
+	config = VolumeTestConfig{
+		Namespace:   namespace,
+		Prefix:      "gluster",
+		ServerImage: imageutils.GetE2EImage(imageutils.VolumeGlusterServer),
+		ServerPorts: []int{24007, 24008, 49152},
+	}
+	pod, ip = CreateStorageServer(cs, config)
+
+	By("creating Gluster endpoints")
+	endpoints := &v1.Endpoints{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Endpoints",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-server",
+		},
+		Subsets: []v1.EndpointSubset{
+			{
+				Addresses: []v1.EndpointAddress{
+					{
+						IP: ip,
+					},
+				},
+				Ports: []v1.EndpointPort{
+					{
+						Name:     "gluster",
+						Port:     24007,
+						Protocol: v1.ProtocolTCP,
+					},
+				},
+			},
+		},
+	}
+	endpoints, err := cs.CoreV1().Endpoints(namespace).Create(endpoints)
+	ExpectNoError(err, "failed to create endpoints for Gluster server")
+
+	return config, pod, ip
+}
+
+// iSCSI-specific wrapper for CreateStorageServer.
+func NewISCSIServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, ip string) {
+	config = VolumeTestConfig{
+		Namespace:   namespace,
+		Prefix:      "iscsi",
+		ServerImage: imageutils.GetE2EImage(imageutils.VolumeISCSIServer),
+		ServerPorts: []int{3260},
+		ServerVolumes: map[string]string{
+			// iSCSI container needs to insert modules from the host
+			"/lib/modules": "/lib/modules",
+		},
+		ServerReadyMessage: "Configuration restored from /etc/target/saveconfig.json",
+	}
+	pod, ip = CreateStorageServer(cs, config)
+	return config, pod, ip
+}
+
+// CephRBD-specific wrapper for CreateStorageServer.
+func NewRBDServer(cs clientset.Interface, namespace string) (config VolumeTestConfig, pod *v1.Pod, secret *v1.Secret, ip string) {
+	config = VolumeTestConfig{
+		Namespace:   namespace,
+		Prefix:      "rbd",
+		ServerImage: imageutils.GetE2EImage(imageutils.VolumeRBDServer),
+		ServerPorts: []int{6789},
+		ServerVolumes: map[string]string{
+			"/lib/modules": "/lib/modules",
+		},
+		ServerReadyMessage: "Ceph is ready",
+	}
+	pod, ip = CreateStorageServer(cs, config)
+	// create secrets for the server
+	secret = &v1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-secret",
+		},
+		Data: map[string][]byte{
+			// from test/images/volumes-tester/rbd/keyring
+			"key": []byte("AQDRrKNVbEevChAAEmRC+pW/KBVHxa0w/POILA=="),
+		},
+		Type: "kubernetes.io/rbd",
+	}
+
+	secret, err := cs.CoreV1().Secrets(config.Namespace).Create(secret)
+	if err != nil {
+		Failf("Failed to create secrets for Ceph RBD: %v", err)
+	}
+
+	return config, pod, secret, ip
+}
+
+// Wrapper for StartVolumeServer(). A storage server config is passed in, and a pod pointer
+// and ip address string are returned.
+// Note: Expect() is called so no error is returned.
+func CreateStorageServer(cs clientset.Interface, config VolumeTestConfig) (pod *v1.Pod, ip string) {
+	pod = StartVolumeServer(cs, config)
+	Expect(pod).NotTo(BeNil(), "storage server pod should not be nil")
+	ip = pod.Status.PodIP
+	Expect(len(ip)).NotTo(BeZero(), fmt.Sprintf("pod %s's IP should not be empty", pod.Name))
+	Logf("%s server pod IP address: %s", config.Prefix, ip)
+	return pod, ip
+}
+
+// Starts a container specified by config.serverImage and exports all
+// config.serverPorts from it. The returned pod should be used to get the server
+// IP address and create appropriate VolumeSource.
+func StartVolumeServer(client clientset.Interface, config VolumeTestConfig) *v1.Pod {
+	podClient := client.CoreV1().Pods(config.Namespace)
+
+	portCount := len(config.ServerPorts)
+	serverPodPorts := make([]v1.ContainerPort, portCount)
+
+	for i := 0; i < portCount; i++ {
+		portName := fmt.Sprintf("%s-%d", config.Prefix, i)
+
+		serverPodPorts[i] = v1.ContainerPort{
+			Name:          portName,
+			ContainerPort: int32(config.ServerPorts[i]),
+			Protocol:      v1.ProtocolTCP,
+		}
+	}
+
+	volumeCount := len(config.ServerVolumes)
+	volumes := make([]v1.Volume, volumeCount)
+	mounts := make([]v1.VolumeMount, volumeCount)
+
+	i := 0
+	for src, dst := range config.ServerVolumes {
+		mountName := fmt.Sprintf("path%d", i)
+		volumes[i].Name = mountName
+		if src == "" {
+			volumes[i].VolumeSource.EmptyDir = &v1.EmptyDirVolumeSource{}
+		} else {
+			volumes[i].VolumeSource.HostPath = &v1.HostPathVolumeSource{
+				Path: src,
+			}
+		}
+
+		mounts[i].Name = mountName
+		mounts[i].ReadOnly = false
+		mounts[i].MountPath = dst
+
+		i++
+	}
+
+	serverPodName := fmt.Sprintf("%s-server", config.Prefix)
+	By(fmt.Sprint("creating ", serverPodName, " pod"))
+	privileged := new(bool)
+	*privileged = true
+
+	restartPolicy := v1.RestartPolicyAlways
+	if config.WaitForCompletion {
+		restartPolicy = v1.RestartPolicyNever
+	}
+	serverPod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: serverPodName,
+			Labels: map[string]string{
+				"role": serverPodName,
+			},
+		},
+
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  serverPodName,
+					Image: config.ServerImage,
+					SecurityContext: &v1.SecurityContext{
+						Privileged: privileged,
+					},
+					Command:      config.ServerCmds,
+					Args:         config.ServerArgs,
+					Ports:        serverPodPorts,
+					VolumeMounts: mounts,
+				},
+			},
+			Volumes:       volumes,
+			RestartPolicy: restartPolicy,
+			NodeName:      config.ServerNodeName,
+			NodeSelector:  config.NodeSelector,
+		},
+	}
+
+	var pod *v1.Pod
+	serverPod, err := podClient.Create(serverPod)
+	// ok if the server pod already exists. TODO: make this controllable by callers
+	if err != nil {
+		if apierrs.IsAlreadyExists(err) {
+			Logf("Ignore \"already-exists\" error, re-get pod...")
+			By(fmt.Sprintf("re-getting the %q server pod", serverPodName))
+			serverPod, err = podClient.Get(serverPodName, metav1.GetOptions{})
+			ExpectNoError(err, "Cannot re-get the server pod %q: %v", serverPodName, err)
+			pod = serverPod
+		} else {
+			ExpectNoError(err, "Failed to create %q pod: %v", serverPodName, err)
+		}
+	}
+	if config.WaitForCompletion {
+		ExpectNoError(WaitForPodSuccessInNamespace(client, serverPod.Name, serverPod.Namespace))
+		ExpectNoError(podClient.Delete(serverPod.Name, nil))
+	} else {
+		ExpectNoError(WaitForPodRunningInNamespace(client, serverPod))
+		if pod == nil {
+			By(fmt.Sprintf("locating the %q server pod", serverPodName))
+			pod, err = podClient.Get(serverPodName, metav1.GetOptions{})
+			ExpectNoError(err, "Cannot locate the server pod %q: %v", serverPodName, err)
+		}
+	}
+	if config.ServerReadyMessage != "" {
+		_, err := LookForStringInLog(pod.Namespace, pod.Name, serverPodName, config.ServerReadyMessage, VolumeServerPodStartupTimeout)
+		ExpectNoError(err, "Failed to find %q in pod logs: %s", config.ServerReadyMessage, err)
+	}
+	return pod
+}
+
+// Wrapper of cleanup function for volume server without secret created by specific CreateStorageServer function.
+func CleanUpVolumeServer(f *Framework, serverPod *v1.Pod) {
+	CleanUpVolumeServerWithSecret(f, serverPod, nil)
+}
+
+// Wrapper of cleanup function for volume server with secret created by specific CreateStorageServer function.
+func CleanUpVolumeServerWithSecret(f *Framework, serverPod *v1.Pod, secret *v1.Secret) {
+	cs := f.ClientSet
+	ns := f.Namespace
+
+	if secret != nil {
+		Logf("Deleting server secret %q...", secret.Name)
+		err := cs.CoreV1().Secrets(ns.Name).Delete(secret.Name, &metav1.DeleteOptions{})
+		if err != nil {
+			Logf("Delete secret failed: %v", err)
+		}
+	}
+
+	Logf("Deleting server pod %q...", serverPod.Name)
+	err := DeletePodWithWait(f, cs, serverPod)
+	if err != nil {
+		Logf("Server pod delete failed: %v", err)
+	}
+}
+
+// Clean both server and client pods.
+func VolumeTestCleanup(f *Framework, config VolumeTestConfig) {
+	By(fmt.Sprint("cleaning the environment after ", config.Prefix))
+
+	defer GinkgoRecover()
+
+	cs := f.ClientSet
+
+	err := DeletePodWithWaitByName(f, cs, config.Prefix+"-client", config.Namespace)
+	Expect(err).To(BeNil(), "Failed to delete pod %v in namespace %v", config.Prefix+"-client", config.Namespace)
+
+	if config.ServerImage != "" {
+		err := DeletePodWithWaitByName(f, cs, config.Prefix+"-server", config.Namespace)
+		Expect(err).To(BeNil(), "Failed to delete pod %v in namespace %v", config.Prefix+"-server", config.Namespace)
+	}
+}
+
+// TestVolumeClient start a client pod using given VolumeSource (exported by startVolumeServer())
+// and check that the pod sees expected data, e.g. from the server pod.
+// Multiple VolumeTests can be specified to mount multiple volumes to a single
+// pod.
+func TestVolumeClient(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, fsType string, tests []VolumeTest) {
+	By(fmt.Sprint("starting ", config.Prefix, "-client"))
+	var gracePeriod int64 = 1
+	var command string
+
+	if !NodeOSDistroIs("windows") {
+		command = "while true ; do cat /opt/0/index.html ; sleep 2 ; ls -altrh /opt/  ; sleep 2 ; done "
+	} else {
+		command = "while(1) {cat /opt/0/index.html ; sleep 2 ; ls /opt/; sleep 2}"
+	}
+	seLinuxOptions := &v1.SELinuxOptions{Level: "s0:c0,c1"}
+	clientPod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: config.Prefix + "-client",
+			Labels: map[string]string{
+				"role": config.Prefix + "-client",
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:       config.Prefix + "-client",
+					Image:      GetTestImage(BusyBoxImage),
+					WorkingDir: "/opt",
+					// An imperative and easily debuggable container which reads vol contents for
+					// us to scan in the tests or by eye.
+					// We expect that /opt is empty in the minimal containers which we use in this test.
+					Command:      GenerateScriptCmd(command),
+					VolumeMounts: []v1.VolumeMount{},
+				},
+			},
+			TerminationGracePeriodSeconds: &gracePeriod,
+			SecurityContext:               GeneratePodSecurityContext(fsGroup, seLinuxOptions),
+			Volumes:                       []v1.Volume{},
+			NodeName:                      config.ClientNodeName,
+			NodeSelector:                  config.NodeSelector,
+		},
+	}
+	podsNamespacer := client.CoreV1().Pods(config.Namespace)
+
+	for i, test := range tests {
+		volumeName := fmt.Sprintf("%s-%s-%d", config.Prefix, "volume", i)
+		clientPod.Spec.Containers[0].VolumeMounts = append(clientPod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
+			Name:      volumeName,
+			MountPath: fmt.Sprintf("/opt/%d", i),
+		})
+		clientPod.Spec.Volumes = append(clientPod.Spec.Volumes, v1.Volume{
+			Name:         volumeName,
+			VolumeSource: test.Volume,
+		})
+	}
+	clientPod, err := podsNamespacer.Create(clientPod)
+	if err != nil {
+		Failf("Failed to create %s pod: %v", clientPod.Name, err)
+
+	}
+	ExpectNoError(WaitForPodRunningInNamespace(client, clientPod))
+
+	By("Checking that text file contents are perfect.")
+	for i, test := range tests {
+		fileName := fmt.Sprintf("/opt/%d/%s", i, test.File)
+		commands := GenerateReadFileCmd(fileName)
+		_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, commands, test.ExpectedContent, time.Minute)
+		ExpectNoError(err, "failed: finding the contents of the mounted file %s.", fileName)
+	}
+	if !NodeOSDistroIs("windows") {
+		if fsGroup != nil {
+			By("Checking fsGroup is correct.")
+			_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"ls", "-ld", "/opt/0"}, strconv.Itoa(int(*fsGroup)), time.Minute)
+			ExpectNoError(err, "failed: getting the right privileges in the file %v", int(*fsGroup))
+		}
+
+		if fsType != "" {
+			By("Checking fsType is correct.")
+			_, err = LookForStringInPodExec(config.Namespace, clientPod.Name, []string{"grep", " /opt/0 ", "/proc/mounts"}, fsType, time.Minute)
+			ExpectNoError(err, "failed: getting the right fsType %s", fsType)
+		}
+	}
+}
+
+// InjectHtml insert index.html with given content into given volume. It does so by
+// starting and auxiliary pod which writes the file there.
+// The volume must be writable.
+func InjectHtml(client clientset.Interface, config VolumeTestConfig, fsGroup *int64, volume v1.VolumeSource, content string) {
+	By(fmt.Sprint("starting ", config.Prefix, " injector"))
+	podClient := client.CoreV1().Pods(config.Namespace)
+	podName := fmt.Sprintf("%s-injector-%s", config.Prefix, rand.String(4))
+	volMountName := fmt.Sprintf("%s-volume-%s", config.Prefix, rand.String(4))
+	fileName := "/mnt/index.html"
+
+	injectPod := &v1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Pod",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: podName,
+			Labels: map[string]string{
+				"role": config.Prefix + "-injector",
+			},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:    config.Prefix + "-injector",
+					Image:   GetTestImage(BusyBoxImage),
+					Command: GenerateWriteFileCmd(content, fileName),
+					VolumeMounts: []v1.VolumeMount{
+						{
+							Name:      volMountName,
+							MountPath: "/mnt",
+						},
+					},
+					SecurityContext: GenerateSecurityContext(true),
+				},
+			},
+			SecurityContext: &v1.PodSecurityContext{
+				FSGroup: fsGroup,
+			},
+			RestartPolicy: v1.RestartPolicyNever,
+			Volumes: []v1.Volume{
+				{
+					Name:         volMountName,
+					VolumeSource: volume,
+				},
+			},
+			NodeName:     config.ClientNodeName,
+			NodeSelector: config.NodeSelector,
+		},
+	}
+
+	defer func() {
+		podClient.Delete(podName, nil)
+	}()
+
+	injectPod, err := podClient.Create(injectPod)
+	ExpectNoError(err, "Failed to create injector pod: %v", err)
+	err = WaitForPodSuccessInNamespace(client, injectPod.Name, injectPod.Namespace)
+	ExpectNoError(err)
+}
+
+func CreateGCEVolume() (*v1.PersistentVolumeSource, string) {
+	diskName, err := CreatePDWithRetry()
+	ExpectNoError(err)
+	return &v1.PersistentVolumeSource{
+		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+			PDName:   diskName,
+			FSType:   "ext3",
+			ReadOnly: false,
+		},
+	}, diskName
+}
+
+// GenerateScriptCmd generates the corresponding command lines to execute a command.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateScriptCmd(command string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"/bin/sh", "-c", command}
+	} else {
+		commands = []string{"powershell", "/c", command}
+	}
+	return commands
+}
+
+// GenerateWriteFileCmd generates the corresponding command lines to write a file with the given content and file path.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateWriteFileCmd(content, fullPath string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"/bin/sh", "-c", "echo '" + content + "' > " + fullPath}
+	} else {
+		commands = []string{"powershell", "/c", "echo '" + content + "' > " + fullPath}
+	}
+	return commands
+}
+
+// GenerateReadFileCmd generates the corresponding command lines to read from a file with the given file path.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateReadFileCmd(fullPath string) []string {
+	var commands []string
+	if !NodeOSDistroIs("windows") {
+		commands = []string{"cat", fullPath}
+	} else {
+		commands = []string{"powershell", "/c", "type " + fullPath}
+	}
+	return commands
+}
+
+// GenerateWriteandExecuteScriptFileCmd generates the corresponding command lines to write a file with the given file path
+// and also execute this file.
+// Depending on the Node OS is Windows or linux, the command will use powershell or /bin/sh
+func GenerateWriteandExecuteScriptFileCmd(content, fileName, filePath string) []string {
+	// for windows cluster, modify the Pod spec.
+	if NodeOSDistroIs("windows") {
+		scriptName := fmt.Sprintf("%s.ps1", fileName)
+		fullPath := filepath.Join(filePath, scriptName)
+
+		cmd := "echo \"" + content + "\" > " + fullPath + "; .\\" + fullPath
+		Logf("generated pod command %s", cmd)
+		return []string{"powershell", "/c", cmd}
+	} else {
+		scriptName := fmt.Sprintf("%s.sh", fileName)
+		fullPath := filepath.Join(filePath, scriptName)
+		cmd := fmt.Sprintf("echo \"%s\" > %s; chmod u+x %s; %s;", content, fullPath, fullPath, fullPath)
+		return []string{"/bin/sh", "-ec", cmd}
+	}
+}
+
+// GenerateSecurityContext generates the corresponding container security context with the given inputs
+// If the Node OS is windows, currently we will ignore the inputs and return nil.
+// TODO: Will modify it after windows has its own security context
+func GenerateSecurityContext(privileged bool) *v1.SecurityContext {
+	if NodeOSDistroIs("windows") {
+		return nil
+	}
+	return &v1.SecurityContext{
+		Privileged: &privileged,
+	}
+}
+
+// GeneratePodSecurityContext generates the corresponding pod security context with the given inputs
+// If the Node OS is windows, currently we will ignore the inputs and return nil.
+// TODO: Will modify it after windows has its own security context
+func GeneratePodSecurityContext(fsGroup *int64, seLinuxOptions *v1.SELinuxOptions) *v1.PodSecurityContext {
+	if NodeOSDistroIs("windows") {
+		return nil
+	}
+	return &v1.PodSecurityContext{
+		SELinuxOptions: seLinuxOptions,
+		FSGroup:        fsGroup,
+	}
+}
+
+// GetTestImage returns the image name with the given input
+// If the Node OS is windows, currently we return Nettest image for Windows node
+// due to the issue of #https://github.com/kubernetes-sigs/windows-testing/pull/35.
+func GetTestImage(image string) string {
+	if NodeOSDistroIs("windows") {
+		return imageutils.GetE2EImage(imageutils.Nettest)
+	}
+	return image
+}