vendor updates

vendor/k8s.io/kubernetes/test/e2e/scheduling/BUILD

@@ -13,6 +13,8 @@ go_library(
         "priorities.go",
         "rescheduler.go",
         "resource_quota.go",
+        "ubernetes_lite.go",
+        "ubernetes_lite_volumes.go",
     ],
     importpath = "k8s.io/kubernetes/test/e2e/scheduling",
     visibility = ["//visibility:public"],
@@ -20,9 +22,11 @@ go_library(
         "//pkg/api/v1/pod:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/extensions:go_default_library",
+        "//pkg/kubelet/apis:go_default_library",
         "//pkg/quota/evaluator/core:go_default_library",
+        "//pkg/scheduler/algorithm/priorities/util:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
         "//pkg/util/version:go_default_library",
-        "//plugin/pkg/scheduler/algorithm/priorities/util:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
         "//test/utils:go_default_library",
@@ -30,6 +34,7 @@ go_library(
         "//vendor/github.com/onsi/ginkgo:go_default_library",
         "//vendor/github.com/onsi/gomega:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/google.golang.org/api/compute/v1:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/scheduling/v1alpha1:go_default_library",
@@ -49,8 +54,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = ["taints_test.go"],
-    importpath = "k8s.io/kubernetes/test/e2e/scheduling",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     deps = [
         "//test/e2e/framework:go_default_library",
         "//test/utils:go_default_library",

vendor/k8s.io/kubernetes/test/e2e/scheduling/equivalence_cache_predicates.go

@@ -155,7 +155,7 @@ var _ = framework.KubeDescribe("EquivalenceCache [Serial]", func() {
 		By("Trying to schedule another equivalent Pod should fail due to node label has been removed.")
 		// use scale to create another equivalent pod and wait for failure event
 		WaitForSchedulerAfterAction(f, func() error {
-			err := framework.ScaleRC(f.ClientSet, f.InternalClientset, ns, affinityRCName, uint(replica+1), false)
+			err := framework.ScaleRC(f.ClientSet, f.InternalClientset, f.ScalesGetter, ns, affinityRCName, uint(replica+1), false)
 			return err
 		}, affinityRCName, false)
 		// and this new pod should be rejected since node label has been updated

vendor/k8s.io/kubernetes/test/e2e/scheduling/nvidia-gpus.go

@@ -17,6 +17,7 @@ limitations under the License.
 package scheduling
 
 import (
+	"os"
 	"strings"
 	"time"
 
@@ -152,7 +153,7 @@ func getGPUsAvailable(f *framework.Framework) int64 {
 	return gpusAvailable
 }
 
-func testNvidiaGPUsOnCOS(f *framework.Framework) {
+func SetupNVIDIAGPUNode(f *framework.Framework, setupResourceGatherer bool) *framework.ContainerResourceGatherer {
 	// Skip the test if the base image is not COS.
 	// TODO: Add support for other base images.
 	// CUDA apps require host mounts which is not portable across base images (yet).
@@ -162,36 +163,46 @@ func testNvidiaGPUsOnCOS(f *framework.Framework) {
 	}
 	framework.Logf("Cluster is running on COS. Proceeding with test")
 
-	if f.BaseName == "device-plugin-gpus" {
-		dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/k8s-1.9/daemonset.yaml"
-		gpuResourceName = framework.NVIDIAGPUResourceName
-		podCreationFunc = makeCudaAdditionDevicePluginTestPod
-	} else {
+	if f.BaseName == "gpus" {
 		dsYamlUrl = "https://raw.githubusercontent.com/ContainerEngine/accelerators/master/cos-nvidia-gpu-installer/daemonset.yaml"
 		gpuResourceName = v1.ResourceNvidiaGPU
 		podCreationFunc = makeCudaAdditionTestPod
+	} else {
+		dsYamlUrlFromEnv := os.Getenv("NVIDIA_DRIVER_INSTALLER_DAEMONSET")
+		if dsYamlUrlFromEnv != "" {
+			dsYamlUrl = dsYamlUrlFromEnv
+		} else {
+			dsYamlUrl = "https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/daemonset.yaml"
+		}
+		gpuResourceName = framework.NVIDIAGPUResourceName
+		podCreationFunc = makeCudaAdditionDevicePluginTestPod
 	}
 
+	framework.Logf("Using %v", dsYamlUrl)
 	// Creates the DaemonSet that installs Nvidia Drivers.
-	// The DaemonSet also runs nvidia device plugin for device plugin test.
 	ds, err := framework.DsFromManifest(dsYamlUrl)
 	Expect(err).NotTo(HaveOccurred())
 	ds.Namespace = f.Namespace.Name
 	_, err = f.ClientSet.ExtensionsV1beta1().DaemonSets(f.Namespace.Name).Create(ds)
-	framework.ExpectNoError(err, "failed to create daemonset")
+	framework.ExpectNoError(err, "failed to create nvidia-driver-installer daemonset")
 	framework.Logf("Successfully created daemonset to install Nvidia drivers.")
 
 	pods, err := framework.WaitForControlledPods(f.ClientSet, ds.Namespace, ds.Name, extensionsinternal.Kind("DaemonSet"))
-	framework.ExpectNoError(err, "getting pods controlled by the daemonset")
+	framework.ExpectNoError(err, "failed to get pods controlled by the nvidia-driver-installer daemonset")
+
 	devicepluginPods, err := framework.WaitForControlledPods(f.ClientSet, "kube-system", "nvidia-gpu-device-plugin", extensionsinternal.Kind("DaemonSet"))
 	if err == nil {
 		framework.Logf("Adding deviceplugin addon pod.")
 		pods.Items = append(pods.Items, devicepluginPods.Items...)
 	}
-	framework.Logf("Starting ResourceUsageGather for the created DaemonSet pods.")
-	rsgather, err := framework.NewResourceUsageGatherer(f.ClientSet, framework.ResourceGathererOptions{false, false, 2 * time.Second, 2 * time.Second, true}, pods)
-	framework.ExpectNoError(err, "creating ResourceUsageGather for the daemonset pods")
-	go rsgather.StartGatheringData()
+
+	var rsgather *framework.ContainerResourceGatherer
+	if setupResourceGatherer {
+		framework.Logf("Starting ResourceUsageGather for the created DaemonSet pods.")
+		rsgather, err = framework.NewResourceUsageGatherer(f.ClientSet, framework.ResourceGathererOptions{false, false, 2 * time.Second, 2 * time.Second, true}, pods)
+		framework.ExpectNoError(err, "creating ResourceUsageGather for the daemonset pods")
+		go rsgather.StartGatheringData()
+	}
 
 	// Wait for Nvidia GPUs to be available on nodes
 	framework.Logf("Waiting for drivers to be installed and GPUs to be available in Node Capacity...")
@@ -199,6 +210,11 @@ func testNvidiaGPUsOnCOS(f *framework.Framework) {
 		return areGPUsAvailableOnAllSchedulableNodes(f)
 	}, driverInstallTimeout, time.Second).Should(BeTrue())
 
+	return rsgather
+}
+
+func testNvidiaGPUsOnCOS(f *framework.Framework) {
+	rsgather := SetupNVIDIAGPUNode(f, true)
 	framework.Logf("Creating as many pods as there are Nvidia GPUs and have the pods run a CUDA app")
 	podList := []*v1.Pod{}
 	for i := int64(0); i < getGPUsAvailable(f); i++ {

vendor/k8s.io/kubernetes/test/e2e/scheduling/preemption.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
 	"k8s.io/kubernetes/test/e2e/framework"
 
 	. "github.com/onsi/ginkgo"
@@ -43,7 +44,6 @@ var _ = SIGDescribe("SchedulerPreemption [Serial] [Feature:PodPreemption]", func
 	lowPriorityClassName := f.BaseName + "-low-priority"
 	mediumPriorityClassName := f.BaseName + "-medium-priority"
 	highPriorityClassName := f.BaseName + "-high-priority"
-
 	AfterEach(func() {
 	})
 
@@ -126,6 +126,66 @@ var _ = SIGDescribe("SchedulerPreemption [Serial] [Feature:PodPreemption]", func
 		}
 	})
 
+	// This test verifies that when a critical pod is created and no node with
+	// enough resources is found, scheduler preempts a lower priority pod to schedule
+	// this critical pod.
+	It("validates lower priority pod preemption by critical pod", func() {
+		var podRes v1.ResourceList
+		// Create one pod per node that uses a lot of the node's resources.
+		By("Create pods that use 60% of node resources.")
+		pods := make([]*v1.Pod, len(nodeList.Items))
+		for i, node := range nodeList.Items {
+			cpuAllocatable, found := node.Status.Allocatable["cpu"]
+			Expect(found).To(Equal(true))
+			milliCPU := cpuAllocatable.MilliValue() * 40 / 100
+			memAllocatable, found := node.Status.Allocatable["memory"]
+			Expect(found).To(Equal(true))
+			memory := memAllocatable.Value() * 60 / 100
+			podRes = v1.ResourceList{}
+			podRes[v1.ResourceCPU] = *resource.NewMilliQuantity(int64(milliCPU), resource.DecimalSI)
+			podRes[v1.ResourceMemory] = *resource.NewQuantity(int64(memory), resource.BinarySI)
+
+			// make the first pod low priority and the rest medium priority.
+			priorityName := mediumPriorityClassName
+			if i == 0 {
+				priorityName = lowPriorityClassName
+			}
+			pods[i] = createPausePod(f, pausePodConfig{
+				Name:              fmt.Sprintf("pod%d-%v", i, priorityName),
+				PriorityClassName: priorityName,
+				Resources: &v1.ResourceRequirements{
+					Requests: podRes,
+				},
+			})
+			framework.Logf("Created pod: %v", pods[i].Name)
+		}
+		By("Wait for pods to be scheduled.")
+		for _, pod := range pods {
+			framework.ExpectNoError(framework.WaitForPodRunningInNamespace(cs, pod))
+		}
+
+		By("Run a critical pod that use 60% of a node resources.")
+		// Create a critical pod and make sure it is scheduled.
+		runPausePod(f, pausePodConfig{
+			Name:              "critical-pod",
+			PriorityClassName: schedulerapi.SystemClusterCritical,
+			Resources: &v1.ResourceRequirements{
+				Requests: podRes,
+			},
+		})
+		// Make sure that the lowest priority pod is deleted.
+		preemptedPod, err := cs.CoreV1().Pods(pods[0].Namespace).Get(pods[0].Name, metav1.GetOptions{})
+		podDeleted := (err != nil && errors.IsNotFound(err)) ||
+			(err == nil && preemptedPod.DeletionTimestamp != nil)
+		Expect(podDeleted).To(BeTrue())
+		// Other pods (mid priority ones) should be present.
+		for i := 1; i < len(pods); i++ {
+			livePod, err := cs.CoreV1().Pods(pods[i].Namespace).Get(pods[i].Name, metav1.GetOptions{})
+			framework.ExpectNoError(err)
+			Expect(livePod.DeletionTimestamp).To(BeNil())
+		}
+	})
+
 	// This test verifies that when a high priority pod is pending and its
 	// scheduling violates a medium priority pod anti-affinity, the medium priority
 	// pod is preempted to allow the higher priority pod schedule.

vendor/k8s.io/kubernetes/test/e2e/scheduling/priorities.go

@@ -29,10 +29,9 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	clientset "k8s.io/client-go/kubernetes"
-	priorityutil "k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities/util"
+	priorityutil "k8s.io/kubernetes/pkg/scheduler/algorithm/priorities/util"
 	"k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
 	testutils "k8s.io/kubernetes/test/utils"
@@ -62,7 +61,6 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
 	var nodeList *v1.NodeList
 	var systemPodsNo int
 	var ns string
-	var masterNodes sets.String
 	f := framework.NewDefaultFramework("sched-priority")
 	ignoreLabels := framework.ImagePullerLabels
 
@@ -75,7 +73,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
 		nodeList = &v1.NodeList{}
 
 		framework.WaitForAllNodesHealthy(cs, time.Minute)
-		masterNodes, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs)
+		_, nodeList = framework.GetMasterAndWorkerNodesOrDie(cs)
 
 		err := framework.CheckTestingNSDeletedExcept(cs, ns)
 		framework.ExpectNoError(err)
@@ -196,7 +194,7 @@ var _ = SIGDescribe("SchedulerPriorities [Serial]", func() {
 
 		By(fmt.Sprintf("Scale the RC: %s to len(nodeList.Item)-1 : %v.", rc.Name, len(nodeList.Items)-1))
 
-		framework.ScaleRC(f.ClientSet, f.InternalClientset, ns, rc.Name, uint(len(nodeList.Items)-1), true)
+		framework.ScaleRC(f.ClientSet, f.InternalClientset, f.ScalesGetter, ns, rc.Name, uint(len(nodeList.Items)-1), true)
 		testPods, err := cs.CoreV1().Pods(ns).List(metav1.ListOptions{
 			LabelSelector: "name=scheduler-priority-avoid-pod",
 		})

vendor/k8s.io/kubernetes/test/e2e/scheduling/rescheduler.go

@@ -42,7 +42,7 @@ var _ = SIGDescribe("Rescheduler [Serial]", func() {
 	var totalMillicores int
 
 	BeforeEach(func() {
-		framework.SkipUnlessProviderIs("gce", "gke")
+		framework.Skipf("Rescheduler is being deprecated soon in Kubernetes 1.10")
 		ns = f.Namespace.Name
 		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
 		nodeCount := len(nodes.Items)
@@ -68,8 +68,8 @@ var _ = SIGDescribe("Rescheduler [Serial]", func() {
 		deployment := deployments.Items[0]
 		replicas := uint(*(deployment.Spec.Replicas))
 
-		err = framework.ScaleDeployment(f.ClientSet, f.InternalClientset, metav1.NamespaceSystem, deployment.Name, replicas+1, true)
-		defer framework.ExpectNoError(framework.ScaleDeployment(f.ClientSet, f.InternalClientset, metav1.NamespaceSystem, deployment.Name, replicas, true))
+		err = framework.ScaleDeployment(f.ClientSet, f.InternalClientset, f.ScalesGetter, metav1.NamespaceSystem, deployment.Name, replicas+1, true)
+		defer framework.ExpectNoError(framework.ScaleDeployment(f.ClientSet, f.InternalClientset, f.ScalesGetter, metav1.NamespaceSystem, deployment.Name, replicas, true))
 		framework.ExpectNoError(err)
 
 	})
@@ -80,7 +80,7 @@ func reserveAllCpu(f *framework.Framework, id string, millicores int) error {
 	replicas := millicores / 100
 
 	reserveCpu(f, id, 1, 100)
-	framework.ExpectNoError(framework.ScaleRC(f.ClientSet, f.InternalClientset, f.Namespace.Name, id, uint(replicas), false))
+	framework.ExpectNoError(framework.ScaleRC(f.ClientSet, f.InternalClientset, f.ScalesGetter, f.Namespace.Name, id, uint(replicas), false))
 
 	for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) {
 		pods, err := framework.GetPodsInNamespace(f.ClientSet, f.Namespace.Name, framework.ImagePullerLabels)

vendor/k8s.io/kubernetes/test/e2e/scheduling/resource_quota.go

@@ -41,6 +41,7 @@ const (
 )
 
 var classGold string = "gold"
+var extendedResourceName string = "example.com/dongle"
 
 var _ = SIGDescribe("ResourceQuota", func() {
 	f := framework.NewDefaultFramework("resourcequota")
@@ -368,9 +369,12 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		By("Creating a Pod that fits quota")
 		podName := "test-pod"
 		requests := v1.ResourceList{}
+		limits := v1.ResourceList{}
 		requests[v1.ResourceCPU] = resource.MustParse("500m")
 		requests[v1.ResourceMemory] = resource.MustParse("252Mi")
-		pod := newTestPodForQuota(f, podName, requests, v1.ResourceList{})
+		requests[v1.ResourceName(extendedResourceName)] = resource.MustParse("2")
+		limits[v1.ResourceName(extendedResourceName)] = resource.MustParse("2")
+		pod := newTestPodForQuota(f, podName, requests, limits)
 		pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
 		Expect(err).NotTo(HaveOccurred())
 		podToUpdate := pod
@@ -380,6 +384,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		usedResources[v1.ResourcePods] = resource.MustParse("1")
 		usedResources[v1.ResourceCPU] = requests[v1.ResourceCPU]
 		usedResources[v1.ResourceMemory] = requests[v1.ResourceMemory]
+		usedResources[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = requests[v1.ResourceName(extendedResourceName)]
 		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -391,6 +396,17 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
 		Expect(err).To(HaveOccurred())
 
+		By("Not allowing a pod to be created that exceeds remaining quota(validation on extended resources)")
+		requests = v1.ResourceList{}
+		limits = v1.ResourceList{}
+		requests[v1.ResourceCPU] = resource.MustParse("500m")
+		requests[v1.ResourceMemory] = resource.MustParse("100Mi")
+		requests[v1.ResourceName(extendedResourceName)] = resource.MustParse("2")
+		limits[v1.ResourceName(extendedResourceName)] = resource.MustParse("2")
+		pod = newTestPodForQuota(f, "fail-pod-for-extended-resource", requests, limits)
+		pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod)
+		Expect(err).To(HaveOccurred())
+
 		By("Ensuring a pod cannot update its resource requirements")
 		// a pod cannot dynamically update its resource requirements.
 		requests = v1.ResourceList{}
@@ -413,6 +429,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		usedResources[v1.ResourcePods] = resource.MustParse("0")
 		usedResources[v1.ResourceCPU] = resource.MustParse("0")
 		usedResources[v1.ResourceMemory] = resource.MustParse("0")
+		usedResources[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = resource.MustParse("0")
 		err = waitForResourceQuota(f.ClientSet, f.Namespace.Name, quotaName, usedResources)
 		Expect(err).NotTo(HaveOccurred())
 	})
@@ -503,7 +520,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 
 		By("Creating a ReplicaSet")
 		replicaSet := newTestReplicaSetForQuota("test-rs", "nginx", 0)
-		replicaSet, err = f.ClientSet.Extensions().ReplicaSets(f.Namespace.Name).Create(replicaSet)
+		replicaSet, err = f.ClientSet.ExtensionsV1beta1().ReplicaSets(f.Namespace.Name).Create(replicaSet)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Ensuring resource quota status captures replicaset creation")
@@ -513,7 +530,7 @@ var _ = SIGDescribe("ResourceQuota", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Deleting a ReplicaSet")
-		err = f.ClientSet.Extensions().ReplicaSets(f.Namespace.Name).Delete(replicaSet.Name, nil)
+		err = f.ClientSet.ExtensionsV1beta1().ReplicaSets(f.Namespace.Name).Delete(replicaSet.Name, nil)
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Ensuring resource quota status released usage")
@@ -833,6 +850,8 @@ func newTestResourceQuota(name string) *v1.ResourceQuota {
 	hard[core.V1ResourceByStorageClass(classGold, v1.ResourceRequestsStorage)] = resource.MustParse("10Gi")
 	// test quota on discovered resource type
 	hard[v1.ResourceName("count/replicasets.extensions")] = resource.MustParse("5")
+	// test quota on extended resource
+	hard[v1.ResourceName(v1.DefaultResourceRequestsPrefix+extendedResourceName)] = resource.MustParse("3")
 	return &v1.ResourceQuota{
 		ObjectMeta: metav1.ObjectMeta{Name: name},
 		Spec:       v1.ResourceQuotaSpec{Hard: hard},

vendor/k8s.io/kubernetes/test/e2e/scheduling/taints_test.go

@@ -147,14 +147,12 @@ const (
 // - lack of eviction of short-tolerating pod after taint removal.
 var _ = SIGDescribe("NoExecuteTaintManager [Serial]", func() {
 	var cs clientset.Interface
-	var nodeList *v1.NodeList
 	var ns string
 	f := framework.NewDefaultFramework("taint-control")
 
 	BeforeEach(func() {
 		cs = f.ClientSet
 		ns = f.Namespace.Name
-		nodeList = &v1.NodeList{}
 
 		framework.WaitForAllNodesHealthy(cs, time.Minute)
 
@@ -196,7 +194,7 @@ var _ = SIGDescribe("NoExecuteTaintManager [Serial]", func() {
 
 	// 1. Run a pod with toleration
 	// 2. Taint the node running this pod with a no-execute taint
-	// 3. See if pod wont get evicted
+	// 3. See if pod won't get evicted
 	It("doesn't evict pod with tolerations from tainted nodes", func() {
 		podName := "taint-eviction-2"
 		pod := createPodForTaintsTest(true, 0, podName, ns)
@@ -228,7 +226,7 @@ var _ = SIGDescribe("NoExecuteTaintManager [Serial]", func() {
 
 	// 1. Run a pod with a finite toleration
 	// 2. Taint the node running this pod with a no-execute taint
-	// 3. See if pod wont get evicted before toleration time runs out
+	// 3. See if pod won't get evicted before toleration time runs out
 	// 4. See if pod will get evicted after toleration time runs out
 	It("eventually evict pod with finite tolerations from tainted nodes", func() {
 		podName := "taint-eviction-3"

vendor/k8s.io/kubernetes/test/e2e/scheduling/ubernetes_lite.go

@@ -0,0 +1,244 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduling
+
+import (
+	"fmt"
+	"math"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	clientset "k8s.io/client-go/kubernetes"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	"k8s.io/kubernetes/test/e2e/framework"
+	testutils "k8s.io/kubernetes/test/utils"
+)
+
+var _ = SIGDescribe("Multi-AZ Clusters", func() {
+	f := framework.NewDefaultFramework("multi-az")
+	var zoneCount int
+	var err error
+	image := framework.ServeHostnameImage
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "gke", "aws")
+		if zoneCount <= 0 {
+			zoneCount, err = getZoneCount(f.ClientSet)
+			Expect(err).NotTo(HaveOccurred())
+		}
+		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
+		msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount)
+		framework.SkipUnlessAtLeast(zoneCount, 2, msg)
+		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
+	})
+	It("should spread the pods of a service across zones", func() {
+		SpreadServiceOrFail(f, (2*zoneCount)+1, image)
+	})
+
+	It("should spread the pods of a replication controller across zones", func() {
+		SpreadRCOrFail(f, int32((2*zoneCount)+1), image)
+	})
+})
+
+// Check that the pods comprising a service get spread evenly across available zones
+func SpreadServiceOrFail(f *framework.Framework, replicaCount int, image string) {
+	// First create the service
+	serviceName := "test-service"
+	serviceSpec := &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      serviceName,
+			Namespace: f.Namespace.Name,
+		},
+		Spec: v1.ServiceSpec{
+			Selector: map[string]string{
+				"service": serviceName,
+			},
+			Ports: []v1.ServicePort{{
+				Port:       80,
+				TargetPort: intstr.FromInt(80),
+			}},
+		},
+	}
+	_, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Create(serviceSpec)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Now create some pods behind the service
+	podSpec := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   serviceName,
+			Labels: map[string]string{"service": serviceName},
+		},
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name:  "test",
+					Image: framework.GetPauseImageName(f.ClientSet),
+				},
+			},
+		},
+	}
+
+	// Caution: StartPods requires at least one pod to replicate.
+	// Based on the callers, replicas is always positive number: zoneCount >= 0 implies (2*zoneCount)+1 > 0.
+	// Thus, no need to test for it. Once the precondition changes to zero number of replicas,
+	// test for replicaCount > 0. Otherwise, StartPods panics.
+	framework.ExpectNoError(testutils.StartPods(f.ClientSet, replicaCount, f.Namespace.Name, serviceName, *podSpec, false, framework.Logf))
+
+	// Wait for all of them to be scheduled
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"service": serviceName}))
+	pods, err := framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, selector)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Now make sure they're spread across zones
+	zoneNames, err := getZoneNames(f.ClientSet)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(checkZoneSpreading(f.ClientSet, pods, zoneNames)).To(Equal(true))
+}
+
+// Find the name of the zone in which a Node is running
+func getZoneNameForNode(node v1.Node) (string, error) {
+	for key, value := range node.Labels {
+		if key == kubeletapis.LabelZoneFailureDomain {
+			return value, nil
+		}
+	}
+	return "", fmt.Errorf("Zone name for node %s not found. No label with key %s",
+		node.Name, kubeletapis.LabelZoneFailureDomain)
+}
+
+// TODO (verult) Merge with framework.GetClusterZones()
+// Find the names of all zones in which we have nodes in this cluster.
+func getZoneNames(c clientset.Interface) ([]string, error) {
+	zoneNames := sets.NewString()
+	nodes, err := c.CoreV1().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	for _, node := range nodes.Items {
+		zoneName, err := getZoneNameForNode(node)
+		Expect(err).NotTo(HaveOccurred())
+		zoneNames.Insert(zoneName)
+	}
+	return zoneNames.List(), nil
+}
+
+// Return the number of zones in which we have nodes in this cluster.
+func getZoneCount(c clientset.Interface) (int, error) {
+	zoneNames, err := getZoneNames(c)
+	if err != nil {
+		return -1, err
+	}
+	return len(zoneNames), nil
+}
+
+// Find the name of the zone in which the pod is scheduled
+func getZoneNameForPod(c clientset.Interface, pod v1.Pod) (string, error) {
+	By(fmt.Sprintf("Getting zone name for pod %s, on node %s", pod.Name, pod.Spec.NodeName))
+	node, err := c.CoreV1().Nodes().Get(pod.Spec.NodeName, metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred())
+	return getZoneNameForNode(*node)
+}
+
+// Determine whether a set of pods are approximately evenly spread
+// across a given set of zones
+func checkZoneSpreading(c clientset.Interface, pods *v1.PodList, zoneNames []string) (bool, error) {
+	podsPerZone := make(map[string]int)
+	for _, zoneName := range zoneNames {
+		podsPerZone[zoneName] = 0
+	}
+	for _, pod := range pods.Items {
+		if pod.DeletionTimestamp != nil {
+			continue
+		}
+		zoneName, err := getZoneNameForPod(c, pod)
+		Expect(err).NotTo(HaveOccurred())
+		podsPerZone[zoneName] = podsPerZone[zoneName] + 1
+	}
+	minPodsPerZone := math.MaxInt32
+	maxPodsPerZone := 0
+	for _, podCount := range podsPerZone {
+		if podCount < minPodsPerZone {
+			minPodsPerZone = podCount
+		}
+		if podCount > maxPodsPerZone {
+			maxPodsPerZone = podCount
+		}
+	}
+	Expect(minPodsPerZone).To(BeNumerically("~", maxPodsPerZone, 1),
+		"Pods were not evenly spread across zones.  %d in one zone and %d in another zone",
+		minPodsPerZone, maxPodsPerZone)
+	return true, nil
+}
+
+// Check that the pods comprising a replication controller get spread evenly across available zones
+func SpreadRCOrFail(f *framework.Framework, replicaCount int32, image string) {
+	name := "ubelite-spread-rc-" + string(uuid.NewUUID())
+	By(fmt.Sprintf("Creating replication controller %s", name))
+	controller, err := f.ClientSet.CoreV1().ReplicationControllers(f.Namespace.Name).Create(&v1.ReplicationController{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: f.Namespace.Name,
+			Name:      name,
+		},
+		Spec: v1.ReplicationControllerSpec{
+			Replicas: &replicaCount,
+			Selector: map[string]string{
+				"name": name,
+			},
+			Template: &v1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{"name": name},
+				},
+				Spec: v1.PodSpec{
+					Containers: []v1.Container{
+						{
+							Name:  name,
+							Image: image,
+							Ports: []v1.ContainerPort{{ContainerPort: 9376}},
+						},
+					},
+				},
+			},
+		},
+	})
+	Expect(err).NotTo(HaveOccurred())
+	// Cleanup the replication controller when we are done.
+	defer func() {
+		// Resize the replication controller to zero to get rid of pods.
+		if err := framework.DeleteRCAndPods(f.ClientSet, f.InternalClientset, f.Namespace.Name, controller.Name); err != nil {
+			framework.Logf("Failed to cleanup replication controller %v: %v.", controller.Name, err)
+		}
+	}()
+	// List the pods, making sure we observe all the replicas.
+	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
+	pods, err := framework.PodsCreated(f.ClientSet, f.Namespace.Name, name, replicaCount)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Wait for all of them to be scheduled
+	By(fmt.Sprintf("Waiting for %d replicas of %s to be scheduled.  Selector: %v", replicaCount, name, selector))
+	pods, err = framework.WaitForPodsWithLabelScheduled(f.ClientSet, f.Namespace.Name, selector)
+	Expect(err).NotTo(HaveOccurred())
+
+	// Now make sure they're spread across zones
+	zoneNames, err := getZoneNames(f.ClientSet)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(checkZoneSpreading(f.ClientSet, pods, zoneNames)).To(Equal(true))
+}

vendor/k8s.io/kubernetes/test/e2e/scheduling/ubernetes_lite_volumes.go

@@ -0,0 +1,265 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package scheduling
+
+import (
+	"fmt"
+	"strconv"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	compute "google.golang.org/api/compute/v1"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+var _ = SIGDescribe("Multi-AZ Cluster Volumes [sig-storage]", func() {
+	f := framework.NewDefaultFramework("multi-az")
+	var zoneCount int
+	var err error
+	image := framework.ServeHostnameImage
+	BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+		if zoneCount <= 0 {
+			zoneCount, err = getZoneCount(f.ClientSet)
+			Expect(err).NotTo(HaveOccurred())
+		}
+		By(fmt.Sprintf("Checking for multi-zone cluster.  Zone count = %d", zoneCount))
+		msg := fmt.Sprintf("Zone count is %d, only run for multi-zone clusters, skipping test", zoneCount)
+		framework.SkipUnlessAtLeast(zoneCount, 2, msg)
+		// TODO: SkipUnlessDefaultScheduler() // Non-default schedulers might not spread
+	})
+	It("should schedule pods in the same zones as statically provisioned PVs", func() {
+		PodsUseStaticPVsOrFail(f, (2*zoneCount)+1, image)
+	})
+
+	It("should only be allowed to provision PDs in zones where nodes exist", func() {
+		OnlyAllowNodeZones(f, zoneCount, image)
+	})
+})
+
+// OnlyAllowNodeZones tests that GetAllCurrentZones returns only zones with Nodes
+func OnlyAllowNodeZones(f *framework.Framework, zoneCount int, image string) {
+	gceCloud, err := framework.GetGCECloud()
+	Expect(err).NotTo(HaveOccurred())
+
+	// Get all the zones that the nodes are in
+	expectedZones, err := gceCloud.GetAllZonesFromCloudProvider()
+	Expect(err).NotTo(HaveOccurred())
+	framework.Logf("Expected zones: %v", expectedZones)
+
+	// Get all the zones in this current region
+	region := gceCloud.Region()
+	allZonesInRegion, err := gceCloud.ListZonesInRegion(region)
+	Expect(err).NotTo(HaveOccurred())
+
+	var extraZone string
+	for _, zone := range allZonesInRegion {
+		if !expectedZones.Has(zone.Name) {
+			extraZone = zone.Name
+			break
+		}
+	}
+	Expect(extraZone).NotTo(Equal(""), fmt.Sprintf("No extra zones available in region %s", region))
+
+	By(fmt.Sprintf("starting a compute instance in unused zone: %v\n", extraZone))
+	project := framework.TestContext.CloudConfig.ProjectID
+	zone := extraZone
+	myuuid := string(uuid.NewUUID())
+	name := "compute-" + myuuid
+	imageURL := "https://www.googleapis.com/compute/v1/projects/debian-cloud/global/images/debian-7-wheezy-v20140606"
+
+	rb := &compute.Instance{
+		MachineType: "zones/" + zone + "/machineTypes/f1-micro",
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				Type:       "PERSISTENT",
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					DiskName:    "my-root-pd-" + myuuid,
+					SourceImage: imageURL,
+				},
+			},
+		},
+		NetworkInterfaces: []*compute.NetworkInterface{
+			{
+				AccessConfigs: []*compute.AccessConfig{
+					{
+						Type: "ONE_TO_ONE_NAT",
+						Name: "External NAT",
+					},
+				},
+				Network: "/global/networks/default",
+			},
+		},
+		Name: name,
+	}
+
+	err = gceCloud.InsertInstance(project, zone, rb)
+	Expect(err).NotTo(HaveOccurred())
+
+	defer func() {
+		// Teardown of the compute instance
+		framework.Logf("Deleting compute resource: %v", name)
+		err := gceCloud.DeleteInstance(project, zone, name)
+		Expect(err).NotTo(HaveOccurred())
+	}()
+
+	By("Creating zoneCount+1 PVCs and making sure PDs are only provisioned in zones with nodes")
+	// Create some (zoneCount+1) PVCs with names of form "pvc-x" where x is 1...zoneCount+1
+	// This will exploit ChooseZoneForVolume in pkg/volume/util.go to provision them in all the zones it "sees"
+	var pvcList []*v1.PersistentVolumeClaim
+	c := f.ClientSet
+	ns := f.Namespace.Name
+
+	for index := 1; index <= zoneCount+1; index++ {
+		pvc := newNamedDefaultClaim(ns, index)
+		pvc, err = framework.CreatePVC(c, ns, pvc)
+		Expect(err).NotTo(HaveOccurred())
+		pvcList = append(pvcList, pvc)
+
+		// Defer the cleanup
+		defer func() {
+			framework.Logf("deleting claim %q/%q", pvc.Namespace, pvc.Name)
+			err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil)
+			if err != nil {
+				framework.Failf("Error deleting claim %q. Error: %v", pvc.Name, err)
+			}
+		}()
+	}
+
+	// Wait for all claims bound
+	for _, claim := range pvcList {
+		err = framework.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, claim.Namespace, claim.Name, framework.Poll, framework.ClaimProvisionTimeout)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	pvZones := sets.NewString()
+	By("Checking that PDs have been provisioned in only the expected zones")
+	for _, claim := range pvcList {
+		// Get a new copy of the claim to have all fields populated
+		claim, err = c.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		// Get the related PV
+		pv, err := c.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+		Expect(err).NotTo(HaveOccurred())
+
+		pvZone, ok := pv.ObjectMeta.Labels[kubeletapis.LabelZoneFailureDomain]
+		Expect(ok).To(BeTrue(), "PV has no LabelZone to be found")
+		pvZones.Insert(pvZone)
+	}
+	Expect(pvZones.Equal(expectedZones)).To(BeTrue(), fmt.Sprintf("PDs provisioned in unwanted zones. We want zones: %v, got: %v", expectedZones, pvZones))
+}
+
+type staticPVTestConfig struct {
+	pvSource *v1.PersistentVolumeSource
+	pv       *v1.PersistentVolume
+	pvc      *v1.PersistentVolumeClaim
+	pod      *v1.Pod
+}
+
+// Check that the pods using statically created PVs get scheduled to the same zone that the PV is in.
+func PodsUseStaticPVsOrFail(f *framework.Framework, podCount int, image string) {
+	var err error
+	c := f.ClientSet
+	ns := f.Namespace.Name
+
+	zones, err := getZoneNames(c)
+	Expect(err).NotTo(HaveOccurred())
+
+	By("Creating static PVs across zones")
+	configs := make([]*staticPVTestConfig, podCount)
+	for i := range configs {
+		configs[i] = &staticPVTestConfig{}
+	}
+
+	defer func() {
+		By("Cleaning up pods and PVs")
+		for _, config := range configs {
+			framework.DeletePodOrFail(c, ns, config.pod.Name)
+		}
+		for _, config := range configs {
+			framework.WaitForPodNoLongerRunningInNamespace(c, config.pod.Name, ns)
+			framework.PVPVCCleanup(c, ns, config.pv, config.pvc)
+			err = framework.DeletePVSource(config.pvSource)
+			Expect(err).NotTo(HaveOccurred())
+		}
+	}()
+
+	for i, config := range configs {
+		zone := zones[i%len(zones)]
+		config.pvSource, err = framework.CreatePVSource(zone)
+		Expect(err).NotTo(HaveOccurred())
+
+		pvConfig := framework.PersistentVolumeConfig{
+			NamePrefix: "multizone-pv",
+			PVSource:   *config.pvSource,
+			Prebind:    nil,
+		}
+		className := ""
+		pvcConfig := framework.PersistentVolumeClaimConfig{StorageClassName: &className}
+
+		config.pv, config.pvc, err = framework.CreatePVPVC(c, pvConfig, pvcConfig, ns, true)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("Waiting for all PVCs to be bound")
+	for _, config := range configs {
+		framework.WaitOnPVandPVC(c, ns, config.pv, config.pvc)
+	}
+
+	By("Creating pods for each static PV")
+	for _, config := range configs {
+		podConfig := framework.MakePod(ns, nil, []*v1.PersistentVolumeClaim{config.pvc}, false, "")
+		config.pod, err = c.CoreV1().Pods(ns).Create(podConfig)
+		Expect(err).NotTo(HaveOccurred())
+	}
+
+	By("Waiting for all pods to be running")
+	for _, config := range configs {
+		err = framework.WaitForPodRunningInNamespace(c, config.pod)
+		Expect(err).NotTo(HaveOccurred())
+	}
+}
+
+func newNamedDefaultClaim(ns string, index int) *v1.PersistentVolumeClaim {
+	claim := v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "pvc-" + strconv.Itoa(index),
+			Namespace: ns,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			AccessModes: []v1.PersistentVolumeAccessMode{
+				v1.ReadWriteOnce,
+			},
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceName(v1.ResourceStorage): resource.MustParse("1Gi"),
+				},
+			},
+		},
+	}
+
+	return &claim
+}