vendor files

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "generic_soak.go",
+        "imports.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/instrumentation/logging",
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/instrumentation/common:go_default_library",
+        "//test/e2e/instrumentation/logging/elasticsearch:go_default_library",
+        "//test/e2e/instrumentation/logging/stackdrvier:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//test/e2e/instrumentation/logging/elasticsearch:all-srcs",
+        "//test/e2e/instrumentation/logging/stackdrvier:all-srcs",
+        "//test/e2e/instrumentation/logging/utils:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/OWNERS

@@ -0,0 +1,8 @@
+reviewers:
+  - coffeepac
+  - crassirostris
+  - piosz
+approvers:
+  - coffeepac
+  - crassirostris
+  - piosz

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "basic.go",
+        "kibana.go",
+        "utils.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch",
+    deps = [
+        "//pkg/apis/core:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/instrumentation/common:go_default_library",
+        "//test/e2e/instrumentation/logging/utils:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/github.com/onsi/gomega:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch/basic.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package elasticsearch
+
+import (
+	"time"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+	"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
+
+	"github.com/onsi/ginkgo"
+)
+
+var _ = instrumentation.SIGDescribe("Cluster level logging using Elasticsearch [Feature:Elasticsearch]", func() {
+	f := framework.NewDefaultFramework("es-logging")
+
+	ginkgo.BeforeEach(func() {
+		// TODO: For now assume we are only testing cluster logging with Elasticsearch
+		// on GCE. Once we are sure that Elasticsearch cluster level logging
+		// works for other providers we should widen this scope of this test.
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	ginkgo.It("should check that logs from containers are ingested into Elasticsearch", func() {
+		ingestionInterval := 10 * time.Second
+		ingestionTimeout := 10 * time.Minute
+
+		p, err := newEsLogProvider(f)
+		framework.ExpectNoError(err, "Failed to create Elasticsearch logs provider")
+
+		err = p.Init()
+		defer p.Cleanup()
+		framework.ExpectNoError(err, "Failed to init Elasticsearch logs provider")
+
+		err = utils.EnsureLoggingAgentDeployment(f, p.LoggingAgentName())
+		framework.ExpectNoError(err, "Fluentd deployed incorrectly")
+
+		pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger", "test"), f)
+		framework.ExpectNoError(err, "Failed to start a pod")
+
+		ginkgo.By("Waiting for logs to ingest")
+		c := utils.NewLogChecker(p, utils.UntilFirstEntry, utils.JustTimeout, pod.Name())
+		err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+		framework.ExpectNoError(err)
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch/kibana.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package elasticsearch
+
+import (
+	"context"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+var _ = instrumentation.SIGDescribe("Kibana Logging Instances Is Alive [Feature:Elasticsearch]", func() {
+	f := framework.NewDefaultFramework("kibana-logging")
+
+	ginkgo.BeforeEach(func() {
+		// TODO: For now assume we are only testing cluster logging with Elasticsearch
+		// and Kibana on GCE. Once we are sure that Elasticsearch and Kibana cluster level logging
+		// works for other providers we should widen this scope of this test.
+		framework.SkipUnlessProviderIs("gce")
+	})
+
+	ginkgo.It("should check that the Kibana logging instance is alive", func() {
+		ClusterLevelLoggingWithKibana(f)
+	})
+})
+
+const (
+	kibanaKey   = "k8s-app"
+	kibanaValue = "kibana-logging"
+)
+
+// ClusterLevelLoggingWithKibana is an end to end test that checks to see if Kibana is alive.
+func ClusterLevelLoggingWithKibana(f *framework.Framework) {
+	const pollingInterval = 10 * time.Second
+	const pollingTimeout = 20 * time.Minute
+
+	// Check for the existence of the Kibana service.
+	ginkgo.By("Checking the Kibana service exists.")
+	s := f.ClientSet.Core().Services(metav1.NamespaceSystem)
+	// Make a few attempts to connect. This makes the test robust against
+	// being run as the first e2e test just after the e2e cluster has been created.
+	err := wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
+		if _, err := s.Get("kibana-logging", metav1.GetOptions{}); err != nil {
+			framework.Logf("Kibana is unreachable: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+
+	// Wait for the Kibana pod(s) to enter the running state.
+	ginkgo.By("Checking to make sure the Kibana pods are running")
+	label := labels.SelectorFromSet(labels.Set(map[string]string{kibanaKey: kibanaValue}))
+	options := metav1.ListOptions{LabelSelector: label.String()}
+	pods, err := f.ClientSet.Core().Pods(metav1.NamespaceSystem).List(options)
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	for _, pod := range pods.Items {
+		err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
+		gomega.Expect(err).NotTo(gomega.HaveOccurred())
+	}
+
+	ginkgo.By("Checking to make sure we get a response from the Kibana UI.")
+	err = wait.Poll(pollingInterval, pollingTimeout, func() (bool, error) {
+		req, err := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
+		if err != nil {
+			framework.Logf("Failed to get services proxy request: %v", err)
+			return false, nil
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), framework.SingleCallTimeout)
+		defer cancel()
+
+		_, err = req.Namespace(metav1.NamespaceSystem).
+			Context(ctx).
+			Name("kibana-logging").
+			DoRaw()
+		if err != nil {
+			framework.Logf("Proxy call to kibana-logging failed: %v", err)
+			return false, nil
+		}
+		return true, nil
+	})
+	gomega.Expect(err).NotTo(gomega.HaveOccurred())
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch/utils.go

@@ -0,0 +1,252 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package elasticsearch
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
+)
+
+const (
+	// esRetryTimeout is how long to keep retrying requesting elasticsearch for status information.
+	esRetryTimeout = 5 * time.Minute
+
+	// esRetryDelay is how much time to wait between two attempts to send a request to elasticsearch
+	esRetryDelay = 5 * time.Second
+
+	// searchPageSize is how many entries to search for in Elasticsearch.
+	searchPageSize = 1000
+)
+
+var _ utils.LogProvider = &esLogProvider{}
+
+type esLogProvider struct {
+	Framework *framework.Framework
+}
+
+func newEsLogProvider(f *framework.Framework) (*esLogProvider, error) {
+	return &esLogProvider{Framework: f}, nil
+}
+
+// Ensures that elasticsearch is running and ready to serve requests
+func (p *esLogProvider) Init() error {
+	f := p.Framework
+	// Check for the existence of the Elasticsearch service.
+	framework.Logf("Checking the Elasticsearch service exists.")
+	s := f.ClientSet.Core().Services(api.NamespaceSystem)
+	// Make a few attempts to connect. This makes the test robust against
+	// being run as the first e2e test just after the e2e cluster has been created.
+	var err error
+	for start := time.Now(); time.Since(start) < esRetryTimeout; time.Sleep(esRetryDelay) {
+		if _, err = s.Get("elasticsearch-logging", meta_v1.GetOptions{}); err == nil {
+			break
+		}
+		framework.Logf("Attempt to check for the existence of the Elasticsearch service failed after %v", time.Since(start))
+	}
+	if err != nil {
+		return err
+	}
+
+	// Wait for the Elasticsearch pods to enter the running state.
+	framework.Logf("Checking to make sure the Elasticsearch pods are running")
+	labelSelector := fields.SelectorFromSet(fields.Set(map[string]string{"k8s-app": "elasticsearch-logging"})).String()
+	options := meta_v1.ListOptions{LabelSelector: labelSelector}
+	pods, err := f.ClientSet.Core().Pods(api.NamespaceSystem).List(options)
+	if err != nil {
+		return err
+	}
+	for _, pod := range pods.Items {
+		err = framework.WaitForPodRunningInNamespace(f.ClientSet, &pod)
+		if err != nil {
+			return err
+		}
+	}
+
+	framework.Logf("Checking to make sure we are talking to an Elasticsearch service.")
+	// Perform a few checks to make sure this looks like an Elasticsearch cluster.
+	var statusCode int
+	err = nil
+	var body []byte
+	for start := time.Now(); time.Since(start) < esRetryTimeout; time.Sleep(esRetryDelay) {
+		proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
+		if errProxy != nil {
+			framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
+			continue
+		}
+		// Query against the root URL for Elasticsearch.
+		response := proxyRequest.Namespace(api.NamespaceSystem).
+			Name("elasticsearch-logging").
+			Do()
+		err = response.Error()
+		response.StatusCode(&statusCode)
+
+		if err != nil {
+			framework.Logf("After %v proxy call to elasticsearch-loigging failed: %v", time.Since(start), err)
+			continue
+		}
+		if int(statusCode) != 200 {
+			framework.Logf("After %v Elasticsearch cluster has a bad status: %v", time.Since(start), statusCode)
+			continue
+		}
+		break
+	}
+	if err != nil {
+		return err
+	}
+	if int(statusCode) != 200 {
+		framework.Failf("Elasticsearch cluster has a bad status: %v", statusCode)
+	}
+
+	// Now assume we really are talking to an Elasticsearch instance.
+	// Check the cluster health.
+	framework.Logf("Checking health of Elasticsearch service.")
+	healthy := false
+	for start := time.Now(); time.Since(start) < esRetryTimeout; time.Sleep(esRetryDelay) {
+		proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
+		if errProxy != nil {
+			framework.Logf("After %v failed to get services proxy request: %v", time.Since(start), errProxy)
+			continue
+		}
+		body, err = proxyRequest.Namespace(api.NamespaceSystem).
+			Name("elasticsearch-logging").
+			Suffix("_cluster/health").
+			Param("level", "indices").
+			DoRaw()
+		if err != nil {
+			continue
+		}
+		health := make(map[string]interface{})
+		err := json.Unmarshal(body, &health)
+		if err != nil {
+			framework.Logf("Bad json response from elasticsearch: %v", err)
+			continue
+		}
+		statusIntf, ok := health["status"]
+		if !ok {
+			framework.Logf("No status field found in cluster health response: %v", health)
+			continue
+		}
+		status := statusIntf.(string)
+		if status != "green" && status != "yellow" {
+			framework.Logf("Cluster health has bad status: %v", health)
+			continue
+		}
+		if err == nil && ok {
+			healthy = true
+			break
+		}
+	}
+	if !healthy {
+		return fmt.Errorf("after %v elasticsearch cluster is not healthy", esRetryTimeout)
+	}
+
+	return nil
+}
+
+func (p *esLogProvider) Cleanup() {
+	// Nothing to do
+}
+
+func (p *esLogProvider) ReadEntries(name string) []utils.LogEntry {
+	f := p.Framework
+
+	proxyRequest, errProxy := framework.GetServicesProxyRequest(f.ClientSet, f.ClientSet.Core().RESTClient().Get())
+	if errProxy != nil {
+		framework.Logf("Failed to get services proxy request: %v", errProxy)
+		return nil
+	}
+
+	query := fmt.Sprintf("kubernetes.pod_name:%s AND kubernetes.namespace_name:%s", name, f.Namespace.Name)
+	framework.Logf("Sending a search request to Elasticsearch with the following query: %s", query)
+
+	// Ask Elasticsearch to return all the log lines that were tagged with the
+	// pod name. Ask for ten times as many log lines because duplication is possible.
+	body, err := proxyRequest.Namespace(api.NamespaceSystem).
+		Name("elasticsearch-logging").
+		Suffix("_search").
+		Param("q", query).
+		// Ask for more in case we included some unrelated records in our query
+		Param("size", strconv.Itoa(searchPageSize)).
+		DoRaw()
+	if err != nil {
+		framework.Logf("Failed to make proxy call to elasticsearch-logging: %v", err)
+		return nil
+	}
+
+	var response map[string]interface{}
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		framework.Logf("Failed to unmarshal response: %v", err)
+		return nil
+	}
+
+	hits, ok := response["hits"].(map[string]interface{})
+	if !ok {
+		framework.Logf("response[hits] not of the expected type: %T", response["hits"])
+		return nil
+	}
+
+	h, ok := hits["hits"].([]interface{})
+	if !ok {
+		framework.Logf("Hits not of the expected type: %T", hits["hits"])
+		return nil
+	}
+
+	entries := []utils.LogEntry{}
+	// Iterate over the hits and populate the observed array.
+	for _, e := range h {
+		l, ok := e.(map[string]interface{})
+		if !ok {
+			framework.Logf("Element of hit not of expected type: %T", e)
+			continue
+		}
+
+		source, ok := l["_source"].(map[string]interface{})
+		if !ok {
+			framework.Logf("_source not of the expected type: %T", l["_source"])
+			continue
+		}
+
+		msg, ok := source["log"].(string)
+		if ok {
+			entries = append(entries, utils.LogEntry{TextPayload: msg})
+			continue
+		}
+
+		obj, ok := source["log"].(map[string]interface{})
+		if ok {
+			entries = append(entries, utils.LogEntry{JSONPayload: obj})
+			continue
+		}
+
+		framework.Logf("Log is of unknown type, got %v, want string or object in field 'log'", source)
+	}
+
+	return entries
+}
+
+func (p *esLogProvider) LoggingAgentName() string {
+	return "fluentd-es"
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/generic_soak.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logging
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/api/core/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+)
+
+var _ = instrumentation.SIGDescribe("Logging soak [Performance] [Slow] [Disruptive]", func() {
+
+	f := framework.NewDefaultFramework("logging-soak")
+
+	// Not a global constant (irrelevant outside this test), also not a parameter (if you want more logs, use --scale=).
+	kbRateInSeconds := 1 * time.Second
+	totalLogTime := 2 * time.Minute
+
+	// This test is designed to run and confirm that logs are being generated at a large scale, and that they can be grabbed by the kubelet.
+	// By running it repeatedly in the background, you can simulate large collections of chatty containers.
+	// This can expose problems in your docker configuration (logging), log searching infrastructure, to tune deployments to match high load
+	// scenarios.  TODO jayunit100 add this to the kube CI in a follow on infra patch.
+
+	// Returns scale (how many waves of pods).
+	// Returns wave interval (how many seconds to wait before dumping the next wave of pods).
+	readConfig := func() (int, time.Duration) {
+		// Read in configuration settings, reasonable defaults.
+		scale := framework.TestContext.LoggingSoak.Scale
+		if framework.TestContext.LoggingSoak.Scale == 0 {
+			scale = 1
+			framework.Logf("Overriding default scale value of zero to %d", scale)
+		}
+
+		milliSecondsBetweenWaves := framework.TestContext.LoggingSoak.MilliSecondsBetweenWaves
+		if milliSecondsBetweenWaves == 0 {
+			milliSecondsBetweenWaves = 5000
+			framework.Logf("Overriding default milliseconds value of zero to %d", milliSecondsBetweenWaves)
+		}
+
+		return scale, time.Duration(milliSecondsBetweenWaves) * time.Millisecond
+	}
+
+	scale, millisecondsBetweenWaves := readConfig()
+	It(fmt.Sprintf("should survive logging 1KB every %v seconds, for a duration of %v, scaling up to %v pods per node", kbRateInSeconds, totalLogTime, scale), func() {
+		defer GinkgoRecover()
+		var wg sync.WaitGroup
+		wg.Add(scale)
+		for i := 0; i < scale; i++ {
+			go func() {
+				wave := fmt.Sprintf("wave%v", strconv.Itoa(i))
+				framework.Logf("Starting logging soak, wave = %v", wave)
+				RunLogPodsWithSleepOf(f, kbRateInSeconds, wave, totalLogTime)
+				framework.Logf("Completed logging soak, wave %v", i)
+				wg.Done()
+			}()
+			// Niceness.
+			time.Sleep(millisecondsBetweenWaves)
+		}
+		framework.Logf("Waiting on all %v logging soak waves to complete", scale)
+		wg.Wait()
+	})
+})
+
+// RunLogPodsWithSleepOf creates a pod on every node, logs continuously (with "sleep" pauses), and verifies that the log string
+// was produced in each and every pod at least once.  The final arg is the timeout for the test to verify all the pods got logs.
+func RunLogPodsWithSleepOf(f *framework.Framework, sleep time.Duration, podname string, timeout time.Duration) {
+
+	nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+	totalPods := len(nodes.Items)
+	Expect(totalPods).NotTo(Equal(0))
+
+	kilobyte := strings.Repeat("logs-123", 128) // 8*128=1024 = 1KB of text.
+
+	appName := "logging-soak" + podname
+	podlables := f.CreatePodsPerNodeForSimpleApp(
+		appName,
+		func(n v1.Node) v1.PodSpec {
+			return v1.PodSpec{
+				Containers: []v1.Container{{
+					Name:  "logging-soak",
+					Image: "busybox",
+					Args: []string{
+						"/bin/sh",
+						"-c",
+						fmt.Sprintf("while true ; do echo %v ; sleep %v; done", kilobyte, sleep.Seconds()),
+					},
+				}},
+				NodeName:      n.Name,
+				RestartPolicy: v1.RestartPolicyAlways,
+			}
+		},
+		totalPods,
+	)
+
+	logSoakVerification := f.NewClusterVerification(
+		f.Namespace,
+		framework.PodStateVerification{
+			Selectors:   podlables,
+			ValidPhases: []v1.PodPhase{v1.PodRunning, v1.PodSucceeded},
+			// we don't validate total log data, since there is no gaurantee all logs will be stored forever.
+			// instead, we just validate that some logs are being created in std out.
+			Verify: func(p v1.Pod) (bool, error) {
+				s, err := framework.LookForStringInLog(f.Namespace.Name, p.Name, "logging-soak", "logs-123", 1*time.Second)
+				return s != "", err
+			},
+		},
+	)
+
+	largeClusterForgiveness := time.Duration(len(nodes.Items)/5) * time.Second // i.e. a 100 node cluster gets an extra 20 seconds to complete.
+	pods, err := logSoakVerification.WaitFor(totalPods, timeout+largeClusterForgiveness)
+
+	if err != nil {
+		framework.Failf("Error in wait... %v", err)
+	} else if len(pods) < totalPods {
+		framework.Failf("Only got %v out of %v", len(pods), totalPods)
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/imports.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package logging
+
+import (
+	_ "k8s.io/kubernetes/test/e2e/instrumentation/logging/elasticsearch"
+	_ "k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier"
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "basic.go",
+        "soak.go",
+        "utils.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier",
+    deps = [
+        "//test/e2e/framework:go_default_library",
+        "//test/e2e/instrumentation/common:go_default_library",
+        "//test/e2e/instrumentation/logging/utils:go_default_library",
+        "//vendor/github.com/onsi/ginkgo:go_default_library",
+        "//vendor/golang.org/x/net/context:go_default_library",
+        "//vendor/golang.org/x/oauth2/google:go_default_library",
+        "//vendor/google.golang.org/api/logging/v2beta1:go_default_library",
+        "//vendor/google.golang.org/api/pubsub/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/json:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier/basic.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stackdriver
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+	"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
+
+	"github.com/onsi/ginkgo"
+	"k8s.io/apimachinery/pkg/util/json"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+const (
+	ingestionInterval = 10 * time.Second
+	ingestionTimeout  = 10 * time.Minute
+)
+
+var _ = instrumentation.SIGDescribe("Cluster level logging implemented by Stackdriver", func() {
+	f := framework.NewDefaultFramework("sd-logging")
+
+	ginkgo.BeforeEach(func() {
+		framework.SkipUnlessProviderIs("gce", "gke")
+	})
+
+	ginkgo.It("should ingest logs", func() {
+		withLogProviderForScope(f, podsScope, func(p *sdLogProvider) {
+			ginkgo.By("Checking ingesting text logs", func() {
+				pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-1", "hey"), f)
+				framework.ExpectNoError(err, "Failed to start a pod")
+
+				ginkgo.By("Waiting for logs to ingest")
+				c := utils.NewLogChecker(p, utils.UntilFirstEntry, utils.JustTimeout, pod.Name())
+				err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+
+			ginkgo.By("Checking ingesting json logs", func() {
+				logRaw := "{\"a\":\"b\"}"
+				pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-2", logRaw), f)
+				framework.ExpectNoError(err, "Failed to start a pod")
+
+				ginkgo.By("Waiting for logs to ingest")
+				c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
+					if len(logEntries) == 0 {
+						return false, nil
+					}
+					log := logEntries[0]
+					if log.JSONPayload == nil {
+						return false, fmt.Errorf("log entry unexpectedly is not json: %s", log.TextPayload)
+					}
+					if log.JSONPayload["a"] != "b" {
+						bytes, err := json.Marshal(log.JSONPayload)
+						if err != nil {
+							return false, fmt.Errorf("log entry ingested incorrectly, failed to marshal: %v", err)
+						}
+						return false, fmt.Errorf("log entry ingested incorrectly, got %v, want %s",
+							string(bytes), logRaw)
+					}
+					return true, nil
+				}, utils.JustTimeout, pod.Name())
+				err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+
+			ginkgo.By("Checking ingesting logs in glog format", func() {
+				logUnformatted := "Text"
+				logRaw := fmt.Sprintf("I0101 00:00:00.000000       1 main.go:1] %s", logUnformatted)
+				pod, err := utils.StartAndReturnSelf(utils.NewRepeatingLoggingPod("synthlogger-3", logRaw), f)
+				framework.ExpectNoError(err, "Failed to start a pod")
+
+				ginkgo.By("Waiting for logs to ingest")
+				c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
+					if len(logEntries) == 0 {
+						return false, nil
+					}
+					log := logEntries[0]
+					if log.TextPayload == "" {
+						return false, fmt.Errorf("log entry is unexpectedly json: %v", log.JSONPayload)
+					}
+					if log.TextPayload != logUnformatted {
+						return false, fmt.Errorf("log entry ingested incorrectly, got %s, want %s",
+							log.TextPayload, logUnformatted)
+					}
+					return true, nil
+				}, utils.JustTimeout, pod.Name())
+				err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+		})
+	})
+
+	ginkgo.It("should ingest logs [Feature:StackdriverLogging]", func() {
+		withLogProviderForScope(f, podsScope, func(p *sdLogProvider) {
+			ginkgo.By("Checking that too long lines are trimmed", func() {
+				originalLength := 100001
+				cmd := []string{
+					"/bin/sh",
+					"-c",
+					fmt.Sprintf("while :; do printf '%%*s' %d | tr ' ' 'A'; echo; sleep 60; done", originalLength),
+				}
+				trimPrefix := "[Trimmed]"
+
+				pod, err := utils.StartAndReturnSelf(utils.NewExecLoggingPod("synthlogger-4", cmd), f)
+				framework.ExpectNoError(err, "Failed to start a pod")
+
+				ginkgo.By("Waiting for logs to ingest")
+				c := utils.NewLogChecker(p, func(_ string, logEntries []utils.LogEntry) (bool, error) {
+					if len(logEntries) == 0 {
+						return false, nil
+					}
+					log := logEntries[0]
+					if log.JSONPayload != nil {
+						return false, fmt.Errorf("got json log entry %v, wanted plain text", log.JSONPayload)
+					}
+					if len(log.TextPayload) == originalLength {
+						return false, fmt.Errorf("got non-trimmed entry of length %d", len(log.TextPayload))
+					}
+					if !strings.HasPrefix(log.TextPayload, trimPrefix) {
+						return false, fmt.Errorf("got message without prefix '%s': %s", trimPrefix, log.TextPayload)
+					}
+					return true, nil
+				}, utils.JustTimeout, pod.Name())
+				err = utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+		})
+	})
+
+	ginkgo.It("should ingest events", func() {
+		eventCreationInterval := 10 * time.Second
+
+		withLogProviderForScope(f, eventsScope, func(p *sdLogProvider) {
+			ginkgo.By("Running pods to generate events while waiting for some of them to be ingested")
+			stopCh := make(chan struct{})
+			cleanupCh := make(chan struct{})
+			defer func() { <-cleanupCh }()
+			defer close(stopCh)
+			go func() {
+				defer ginkgo.GinkgoRecover()
+				defer close(cleanupCh)
+
+				wait.PollUntil(eventCreationInterval, func() (bool, error) {
+					podName := fmt.Sprintf("synthlogger-%s", string(uuid.NewUUID()))
+					err := utils.NewLoadLoggingPod(podName, "", 1, 1*time.Second).Start(f)
+					if err != nil {
+						framework.Logf("Failed to create a logging pod: %v", err)
+					}
+					return false, nil
+				}, stopCh)
+			}()
+
+			ginkgo.By("Waiting for events to ingest")
+			c := utils.NewLogChecker(p, utils.UntilFirstEntry, utils.JustTimeout, "")
+			err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+			framework.ExpectNoError(err)
+		})
+	})
+
+	ginkgo.It("should ingest system logs from all nodes [Feature:StackdriverLogging]", func() {
+		withLogProviderForScope(f, systemScope, func(p *sdLogProvider) {
+			ginkgo.By("Waiting for some kubelet logs to be ingested from each node", func() {
+				nodeIds := utils.GetNodeIds(f.ClientSet)
+				log := fmt.Sprintf("projects/%s/logs/kubelet", framework.TestContext.CloudConfig.ProjectID)
+				c := utils.NewLogChecker(p, utils.UntilFirstEntryFromLog(log), utils.JustTimeout, nodeIds...)
+				err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+
+			ginkgo.By("Waiting for some docker logs to be ingested from each node", func() {
+				nodeIds := utils.GetNodeIds(f.ClientSet)
+				log := fmt.Sprintf("projects/%s/logs/docker", framework.TestContext.CloudConfig.ProjectID)
+				c := utils.NewLogChecker(p, utils.UntilFirstEntryFromLog(log), utils.JustTimeout, nodeIds...)
+				err := utils.WaitForLogs(c, ingestionInterval, ingestionTimeout)
+				framework.ExpectNoError(err)
+			})
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier/soak.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stackdriver
+
+import (
+	"fmt"
+	"math"
+	"time"
+
+	"k8s.io/kubernetes/test/e2e/framework"
+	instrumentation "k8s.io/kubernetes/test/e2e/instrumentation/common"
+	"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
+
+	"github.com/onsi/ginkgo"
+)
+
+const (
+	// maxAllowedLostFraction is the fraction of lost logs considered acceptable.
+	maxAllowedLostFraction = 0.01
+	// maxAllowedRestartsPerHour is the number of fluentd container restarts
+	// considered acceptable. Once per hour is fine for now, as long as it
+	// doesn't loose too much logs.
+	maxAllowedRestartsPerHour = 1.0
+	// lastPodIngestionSlack is the amount of time to wait for the last pod's
+	// logs to be ingested by the logging agent.
+	lastPodIngestionSlack = 5 * time.Minute
+)
+
+var _ = instrumentation.SIGDescribe("Cluster level logging implemented by Stackdriver [Feature:StackdriverLogging] [Soak]", func() {
+	f := framework.NewDefaultFramework("sd-logging-load")
+
+	ginkgo.It("should ingest logs from applications running for a prolonged amount of time", func() {
+		withLogProviderForScope(f, podsScope, func(p *sdLogProvider) {
+			nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
+			maxPodCount := 10
+			jobDuration := 30 * time.Minute
+			linesPerPodPerSecond := 100
+			// TODO(crassirostris): Increase to 21 hrs
+			testDuration := 3 * time.Hour
+			ingestionInterval := 1 * time.Minute
+			ingestionTimeout := testDuration + 30*time.Minute
+			allowedRestarts := int(math.Ceil(float64(testDuration) /
+				float64(time.Hour) * maxAllowedRestartsPerHour))
+
+			podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
+			podRunCount := maxPodCount*(int(testDuration/jobDuration)-1) + 1
+			linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds())
+
+			// pods is a flat array of all pods to be run and to expect in Stackdriver.
+			pods := []utils.FiniteLoggingPod{}
+			// podsByRun is a two-dimensional array of pods, first dimension is the run
+			// index, the second dimension is the node index. Since we want to create
+			// an equal load on all nodes, for the same run we have one pod per node.
+			podsByRun := [][]utils.FiniteLoggingPod{}
+			for runIdx := 0; runIdx < podRunCount; runIdx++ {
+				podsInRun := []utils.FiniteLoggingPod{}
+				for nodeIdx, node := range nodes {
+					podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx)
+					pod := utils.NewLoadLoggingPod(podName, node.Name, linesPerPod, jobDuration)
+					pods = append(pods, pod)
+					podsInRun = append(podsInRun, pod)
+				}
+				podsByRun = append(podsByRun, podsInRun)
+			}
+
+			ginkgo.By("Running short-living pods")
+			go func() {
+				t := time.NewTicker(podRunDelay)
+				defer t.Stop()
+				for runIdx := 0; runIdx < podRunCount; runIdx++ {
+					// Starting one pod on each node.
+					for _, pod := range podsByRun[runIdx] {
+						if err := pod.Start(f); err != nil {
+							framework.Logf("Failed to start pod: %v", err)
+						}
+					}
+					<-t.C
+				}
+			}()
+
+			checker := utils.NewFullIngestionPodLogChecker(p, maxAllowedLostFraction, pods...)
+			err := utils.WaitForLogs(checker, ingestionInterval, ingestionTimeout)
+			framework.ExpectNoError(err)
+
+			utils.EnsureLoggingAgentRestartsCount(f, p.LoggingAgentName(), allowedRestarts)
+		})
+	})
+})

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/stackdrvier/utils.go

@@ -0,0 +1,386 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package stackdriver
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/instrumentation/logging/utils"
+
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2/google"
+	sd "google.golang.org/api/logging/v2beta1"
+	pubsub "google.golang.org/api/pubsub/v1"
+)
+
+const (
+	// The amount of time to wait for Stackdriver Logging
+	// sink to become operational
+	sinkStartupTimeout = 10 * time.Minute
+
+	// The limit on the number of messages to pull from PubSub
+	maxPullLogMessages = 100 * 1000
+
+	// maxQueueSize is the limit on the number of messages in the single queue.
+	maxQueueSize = 10 * 1000
+
+	// PubSub topic with log entries polling interval
+	sdLoggingPollInterval = 100 * time.Millisecond
+)
+
+type logProviderScope int
+
+const (
+	podsScope logProviderScope = iota
+	eventsScope
+	systemScope
+)
+
+var _ utils.LogProvider = &sdLogProvider{}
+
+type sdLogProvider struct {
+	sdService     *sd.Service
+	pubsubService *pubsub.Service
+
+	framework *framework.Framework
+
+	topic        *pubsub.Topic
+	subscription *pubsub.Subscription
+	logSink      *sd.LogSink
+
+	pollingStopChannel chan struct{}
+
+	queueCollection utils.LogsQueueCollection
+
+	scope logProviderScope
+}
+
+func newSdLogProvider(f *framework.Framework, scope logProviderScope) (*sdLogProvider, error) {
+	ctx := context.Background()
+	hc, err := google.DefaultClient(ctx, sd.CloudPlatformScope)
+	sdService, err := sd.New(hc)
+	if err != nil {
+		return nil, err
+	}
+
+	pubsubService, err := pubsub.New(hc)
+	if err != nil {
+		return nil, err
+	}
+
+	provider := &sdLogProvider{
+		scope:              scope,
+		sdService:          sdService,
+		pubsubService:      pubsubService,
+		framework:          f,
+		pollingStopChannel: make(chan struct{}, 1),
+		queueCollection:    utils.NewLogsQueueCollection(maxQueueSize),
+	}
+	return provider, nil
+}
+
+func (p *sdLogProvider) Init() error {
+	projectID := framework.TestContext.CloudConfig.ProjectID
+	nsName := p.framework.Namespace.Name
+
+	topic, err := p.createPubSubTopic(projectID, nsName)
+	if err != nil {
+		return fmt.Errorf("failed to create PubSub topic: %v", err)
+	}
+	p.topic = topic
+
+	subs, err := p.createPubSubSubscription(projectID, nsName, topic.Name)
+	if err != nil {
+		return fmt.Errorf("failed to create PubSub subscription: %v", err)
+	}
+	p.subscription = subs
+
+	logSink, err := p.createSink(projectID, nsName, topic.Name)
+	if err != nil {
+		return fmt.Errorf("failed to create Stackdriver Logging sink: %v", err)
+	}
+	p.logSink = logSink
+
+	if err = p.authorizeSink(); err != nil {
+		return fmt.Errorf("failed to authorize log sink: %v", err)
+	}
+
+	if err = p.waitSinkInit(); err != nil {
+		return fmt.Errorf("failed to wait for sink to become operational: %v", err)
+	}
+
+	go p.pollLogs()
+
+	return nil
+}
+
+func (p *sdLogProvider) Cleanup() {
+	p.pollingStopChannel <- struct{}{}
+
+	if p.logSink != nil {
+		projectID := framework.TestContext.CloudConfig.ProjectID
+		sinkNameID := fmt.Sprintf("projects/%s/sinks/%s", projectID, p.logSink.Name)
+		sinksService := p.sdService.Projects.Sinks
+		if _, err := sinksService.Delete(sinkNameID).Do(); err != nil {
+			framework.Logf("Failed to delete LogSink: %v", err)
+		}
+	}
+
+	if p.subscription != nil {
+		subsService := p.pubsubService.Projects.Subscriptions
+		if _, err := subsService.Delete(p.subscription.Name).Do(); err != nil {
+			framework.Logf("Failed to delete PubSub subscription: %v", err)
+		}
+	}
+
+	if p.topic != nil {
+		topicsService := p.pubsubService.Projects.Topics
+		if _, err := topicsService.Delete(p.topic.Name).Do(); err != nil {
+			framework.Logf("Failed to delete PubSub topic: %v", err)
+		}
+	}
+}
+
+func (p *sdLogProvider) ReadEntries(name string) []utils.LogEntry {
+	return p.queueCollection.Pop(name)
+}
+
+func (p *sdLogProvider) LoggingAgentName() string {
+	return "fluentd-gcp"
+}
+
+func (p *sdLogProvider) createPubSubTopic(projectID, topicName string) (*pubsub.Topic, error) {
+	topicFullName := fmt.Sprintf("projects/%s/topics/%s", projectID, topicName)
+	topic := &pubsub.Topic{
+		Name: topicFullName,
+	}
+	return p.pubsubService.Projects.Topics.Create(topicFullName, topic).Do()
+}
+
+func (p *sdLogProvider) createPubSubSubscription(projectID, subsName, topicName string) (*pubsub.Subscription, error) {
+	subsFullName := fmt.Sprintf("projects/%s/subscriptions/%s", projectID, subsName)
+	subs := &pubsub.Subscription{
+		Name:  subsFullName,
+		Topic: topicName,
+	}
+	return p.pubsubService.Projects.Subscriptions.Create(subsFullName, subs).Do()
+}
+
+func (p *sdLogProvider) createSink(projectID, sinkName, topicName string) (*sd.LogSink, error) {
+	filter, err := p.buildFilter()
+	if err != nil {
+		return nil, err
+	}
+	framework.Logf("Using the following filter for log entries: %s", filter)
+	sink := &sd.LogSink{
+		Name:        sinkName,
+		Destination: fmt.Sprintf("pubsub.googleapis.com/%s", topicName),
+		Filter:      filter,
+	}
+	projectDst := fmt.Sprintf("projects/%s", projectID)
+	return p.sdService.Projects.Sinks.Create(projectDst, sink).Do()
+}
+
+func (p *sdLogProvider) buildFilter() (string, error) {
+	switch p.scope {
+	case podsScope:
+		return fmt.Sprintf("resource.type=\"container\" AND resource.labels.namespace_id=\"%s\"",
+			p.framework.Namespace.Name), nil
+	case eventsScope:
+		return fmt.Sprintf("resource.type=\"gke_cluster\" AND jsonPayload.metadata.namespace=\"%s\"",
+			p.framework.Namespace.Name), nil
+	case systemScope:
+		// TODO(instrumentation): Filter logs from the current project only.
+		return "resource.type=\"gce_instance\"", nil
+	}
+	return "", fmt.Errorf("Unknown log provider scope: %v", p.scope)
+}
+
+func (p *sdLogProvider) authorizeSink() error {
+	topicsService := p.pubsubService.Projects.Topics
+	policy, err := topicsService.GetIamPolicy(p.topic.Name).Do()
+	if err != nil {
+		return err
+	}
+
+	binding := &pubsub.Binding{
+		Role:    "roles/pubsub.publisher",
+		Members: []string{p.logSink.WriterIdentity},
+	}
+	policy.Bindings = append(policy.Bindings, binding)
+	req := &pubsub.SetIamPolicyRequest{Policy: policy}
+	if _, err = topicsService.SetIamPolicy(p.topic.Name, req).Do(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (p *sdLogProvider) waitSinkInit() error {
+	framework.Logf("Waiting for log sink to become operational")
+	return wait.Poll(1*time.Second, sinkStartupTimeout, func() (bool, error) {
+		err := publish(p.pubsubService, p.topic, "embrace eternity")
+		if err != nil {
+			framework.Logf("Failed to push message to PubSub due to %v", err)
+		}
+
+		messages, err := pullAndAck(p.pubsubService, p.subscription)
+		if err != nil {
+			framework.Logf("Failed to pull messages from PubSub due to %v", err)
+			return false, nil
+		}
+		if len(messages) > 0 {
+			framework.Logf("Sink %s is operational", p.logSink.Name)
+			return true, nil
+		}
+
+		return false, nil
+	})
+}
+
+func (p *sdLogProvider) pollLogs() {
+	wait.PollUntil(sdLoggingPollInterval, func() (bool, error) {
+		messages, err := pullAndAck(p.pubsubService, p.subscription)
+		if err != nil {
+			framework.Logf("Failed to pull messages from PubSub due to %v", err)
+			return false, nil
+		}
+
+		for _, msg := range messages {
+			logEntryEncoded, err := base64.StdEncoding.DecodeString(msg.Message.Data)
+			if err != nil {
+				framework.Logf("Got a message from pubsub that is not base64-encoded: %s", msg.Message.Data)
+				continue
+			}
+
+			var sdLogEntry sd.LogEntry
+			if err := json.Unmarshal(logEntryEncoded, &sdLogEntry); err != nil {
+				framework.Logf("Failed to decode a pubsub message '%s': %v", logEntryEncoded, err)
+				continue
+			}
+
+			name, ok := p.tryGetName(sdLogEntry)
+			if !ok {
+				framework.Logf("Received LogEntry with unexpected resource type: %s", sdLogEntry.Resource.Type)
+				continue
+			}
+
+			logEntry, err := convertLogEntry(sdLogEntry)
+			if err != nil {
+				framework.Logf("Failed to parse Stackdriver LogEntry: %v", err)
+				continue
+			}
+
+			p.queueCollection.Push(name, logEntry)
+		}
+
+		return false, nil
+	}, p.pollingStopChannel)
+}
+
+func (p *sdLogProvider) tryGetName(sdLogEntry sd.LogEntry) (string, bool) {
+	switch sdLogEntry.Resource.Type {
+	case "container":
+		return sdLogEntry.Resource.Labels["pod_id"], true
+	case "gke_cluster":
+		return "", true
+	case "gce_instance":
+		return sdLogEntry.Resource.Labels["instance_id"], true
+	}
+	return "", false
+}
+
+func convertLogEntry(sdLogEntry sd.LogEntry) (entry utils.LogEntry, err error) {
+	entry = utils.LogEntry{LogName: sdLogEntry.LogName}
+	if sdLogEntry.TextPayload != "" {
+		entry.TextPayload = sdLogEntry.TextPayload
+		return
+	}
+
+	bytes, err := sdLogEntry.JsonPayload.MarshalJSON()
+	if err != nil {
+		err = fmt.Errorf("Failed to get jsonPayload from LogEntry %v", sdLogEntry)
+		return
+	}
+
+	var jsonObject map[string]interface{}
+	err = json.Unmarshal(bytes, &jsonObject)
+	if err != nil {
+		err = fmt.Errorf("Failed to deserialize jsonPayload as json object %s", string(bytes[:]))
+		return
+	}
+	entry.JSONPayload = jsonObject
+	return
+}
+
+func pullAndAck(service *pubsub.Service, subs *pubsub.Subscription) ([]*pubsub.ReceivedMessage, error) {
+	subsService := service.Projects.Subscriptions
+	req := &pubsub.PullRequest{
+		ReturnImmediately: true,
+		MaxMessages:       maxPullLogMessages,
+	}
+
+	resp, err := subsService.Pull(subs.Name, req).Do()
+	if err != nil {
+		return nil, err
+	}
+
+	var ids []string
+	for _, msg := range resp.ReceivedMessages {
+		ids = append(ids, msg.AckId)
+	}
+	if len(ids) > 0 {
+		ackReq := &pubsub.AcknowledgeRequest{AckIds: ids}
+		if _, err = subsService.Acknowledge(subs.Name, ackReq).Do(); err != nil {
+			framework.Logf("Failed to ack poll: %v", err)
+		}
+	}
+
+	return resp.ReceivedMessages, nil
+}
+
+func publish(service *pubsub.Service, topic *pubsub.Topic, msg string) error {
+	topicsService := service.Projects.Topics
+	req := &pubsub.PublishRequest{
+		Messages: []*pubsub.PubsubMessage{
+			{
+				Data: base64.StdEncoding.EncodeToString([]byte(msg)),
+			},
+		},
+	}
+	_, err := topicsService.Publish(topic.Name, req).Do()
+	return err
+}
+
+func withLogProviderForScope(f *framework.Framework, scope logProviderScope, fun func(*sdLogProvider)) {
+	p, err := newSdLogProvider(f, scope)
+	framework.ExpectNoError(err, "Failed to create Stackdriver logs provider")
+
+	err = p.Init()
+	defer p.Cleanup()
+	framework.ExpectNoError(err, "Failed to init Stackdriver logs provider")
+
+	err = utils.EnsureLoggingAgentDeployment(f, p.LoggingAgentName())
+	framework.ExpectNoError(err, "Logging agents deployed incorrectly")
+
+	fun(p)
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "log_provider.go",
+        "logging_agent.go",
+        "logging_pod.go",
+        "misc.go",
+        "types.go",
+        "wait.go",
+    ],
+    importpath = "k8s.io/kubernetes/test/e2e/instrumentation/logging/utils",
+    deps = [
+        "//pkg/apis/core:go_default_library",
+        "//test/e2e/framework:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/api/resource:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/labels:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/util/integer:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/log_provider.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+// LogProvider interface provides an API to get logs from the logging backend.
+type LogProvider interface {
+	Init() error
+	Cleanup()
+	ReadEntries(name string) []LogEntry
+	LoggingAgentName() string
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/logging_agent.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+
+	api_v1 "k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/util/integer"
+	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// EnsureLoggingAgentDeployment checks that logging agent is present on each
+// node and returns an error if that's not true.
+func EnsureLoggingAgentDeployment(f *framework.Framework, appName string) error {
+	agentPods, err := getLoggingAgentPods(f, appName)
+	if err != nil {
+		return fmt.Errorf("failed to get logging agent pods: %v", err)
+	}
+
+	agentPerNode := make(map[string]int)
+	for _, pod := range agentPods.Items {
+		agentPerNode[pod.Spec.NodeName]++
+	}
+
+	nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
+	for _, node := range nodeList.Items {
+		agentPodsCount, ok := agentPerNode[node.Name]
+
+		if !ok {
+			return fmt.Errorf("node %s doesn't have logging agents, want 1", node.Name)
+		} else if agentPodsCount != 1 {
+			return fmt.Errorf("node %s has %d logging agents, want 1", node.Name, agentPodsCount)
+		}
+	}
+
+	return nil
+}
+
+// EnsureLoggingAgentRestartsCount checks that each logging agent was restarted
+// no more than maxRestarts times and returns an error if there's a pod which
+// exceeds this number of restarts.
+func EnsureLoggingAgentRestartsCount(f *framework.Framework, appName string, maxRestarts int) error {
+	agentPods, err := getLoggingAgentPods(f, appName)
+	if err != nil {
+		return fmt.Errorf("failed to get logging agent pods: %v", err)
+	}
+
+	maxRestartCount := 0
+	for _, pod := range agentPods.Items {
+		contStatuses := pod.Status.ContainerStatuses
+		if len(contStatuses) == 0 {
+			framework.Logf("There are no container statuses for pod %s", pod.Name)
+			continue
+		}
+		restartCount := int(contStatuses[0].RestartCount)
+		maxRestartCount = integer.IntMax(maxRestartCount, restartCount)
+
+		framework.Logf("Logging agent %s on node %s was restarted %d times",
+			pod.Name, pod.Spec.NodeName, restartCount)
+	}
+
+	if maxRestartCount > maxRestarts {
+		return fmt.Errorf("max logging agent restarts was %d, which is more than allowed %d",
+			maxRestartCount, maxRestarts)
+	}
+	return nil
+}
+
+func getLoggingAgentPods(f *framework.Framework, appName string) (*api_v1.PodList, error) {
+	label := labels.SelectorFromSet(labels.Set(map[string]string{"k8s-app": appName}))
+	options := meta_v1.ListOptions{LabelSelector: label.String()}
+	return f.ClientSet.CoreV1().Pods(api.NamespaceSystem).List(options)
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/logging_pod.go

@@ -0,0 +1,195 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"strconv"
+	"time"
+
+	"fmt"
+	api_v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	// Amount of requested cores for logging container in millicores
+	loggingContainerCPURequest = 10
+
+	// Amount of requested memory for logging container in bytes
+	loggingContainerMemoryRequest = 10 * 1024 * 1024
+
+	// Name of the container used for logging tests
+	loggingContainerName = "logging-container"
+)
+
+// LoggingPod is an interface of a pod that can be started and that logs
+// something to its stdout, possibly indefinitely.
+type LoggingPod interface {
+	// Name equals to the Kubernetes pod name.
+	Name() string
+
+	// Start method controls when the logging pod is started in the cluster.
+	Start(f *framework.Framework) error
+}
+
+// StartAndReturnSelf is a helper method to start a logging pod and
+// immediately return it.
+func StartAndReturnSelf(p LoggingPod, f *framework.Framework) (LoggingPod, error) {
+	err := p.Start(f)
+	return p, err
+}
+
+// FiniteLoggingPod is a logging pod that emits a known number of log lines.
+type FiniteLoggingPod interface {
+	LoggingPod
+
+	// ExpectedLinesNumber returns the number of lines that are
+	// expected to be ingested from this pod.
+	ExpectedLineCount() int
+}
+
+var _ FiniteLoggingPod = &loadLoggingPod{}
+
+type loadLoggingPod struct {
+	name               string
+	nodeName           string
+	expectedLinesCount int
+	runDuration        time.Duration
+}
+
+// NewLoadLoggingPod returns a logging pod that generates totalLines random
+// lines over period of length loggingDuration. Lines generated by this
+// pod are numbered and have well-defined structure.
+func NewLoadLoggingPod(podName string, nodeName string, totalLines int,
+	loggingDuration time.Duration) FiniteLoggingPod {
+	return &loadLoggingPod{
+		name:               podName,
+		nodeName:           nodeName,
+		expectedLinesCount: totalLines,
+		runDuration:        loggingDuration,
+	}
+}
+
+func (p *loadLoggingPod) Name() string {
+	return p.name
+}
+
+func (p *loadLoggingPod) Start(f *framework.Framework) error {
+	framework.Logf("Starting load logging pod %s", p.name)
+	f.PodClient().Create(&api_v1.Pod{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name: p.name,
+		},
+		Spec: api_v1.PodSpec{
+			RestartPolicy: api_v1.RestartPolicyNever,
+			Containers: []api_v1.Container{
+				{
+					Name:  loggingContainerName,
+					Image: "gcr.io/google_containers/logs-generator:v0.1.0",
+					Env: []api_v1.EnvVar{
+						{
+							Name:  "LOGS_GENERATOR_LINES_TOTAL",
+							Value: strconv.Itoa(p.expectedLinesCount),
+						},
+						{
+							Name:  "LOGS_GENERATOR_DURATION",
+							Value: p.runDuration.String(),
+						},
+					},
+					Resources: api_v1.ResourceRequirements{
+						Requests: api_v1.ResourceList{
+							api_v1.ResourceCPU: *resource.NewMilliQuantity(
+								loggingContainerCPURequest,
+								resource.DecimalSI),
+							api_v1.ResourceMemory: *resource.NewQuantity(
+								loggingContainerMemoryRequest,
+								resource.BinarySI),
+						},
+					},
+				},
+			},
+			NodeName: p.nodeName,
+		},
+	})
+	return framework.WaitForPodNameRunningInNamespace(f.ClientSet, p.name, f.Namespace.Name)
+}
+
+func (p *loadLoggingPod) ExpectedLineCount() int {
+	return p.expectedLinesCount
+}
+
+// NewRepeatingLoggingPod returns a logging pod that each second prints
+// line value to its stdout.
+func NewRepeatingLoggingPod(podName string, line string) LoggingPod {
+	cmd := []string{
+		"/bin/sh",
+		"-c",
+		fmt.Sprintf("while :; do echo '%s'; sleep 1; done", line),
+	}
+	return NewExecLoggingPod(podName, cmd)
+}
+
+var _ LoggingPod = &execLoggingPod{}
+
+type execLoggingPod struct {
+	name string
+	cmd  []string
+}
+
+// NewExecLoggingPod returns a logging pod that produces logs through
+// executing a command, passed in cmd.
+func NewExecLoggingPod(podName string, cmd []string) LoggingPod {
+	return &execLoggingPod{
+		name: podName,
+		cmd:  cmd,
+	}
+}
+
+func (p *execLoggingPod) Name() string {
+	return p.name
+}
+
+func (p *execLoggingPod) Start(f *framework.Framework) error {
+	framework.Logf("Starting repeating logging pod %s", p.name)
+	f.PodClient().Create(&api_v1.Pod{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name: p.name,
+		},
+		Spec: api_v1.PodSpec{
+			Containers: []api_v1.Container{
+				{
+					Name:    loggingContainerName,
+					Image:   "busybox",
+					Command: p.cmd,
+					Resources: api_v1.ResourceRequirements{
+						Requests: api_v1.ResourceList{
+							api_v1.ResourceCPU: *resource.NewMilliQuantity(
+								loggingContainerCPURequest,
+								resource.DecimalSI),
+							api_v1.ResourceMemory: *resource.NewQuantity(
+								loggingContainerMemoryRequest,
+								resource.BinarySI),
+						},
+					},
+				},
+			},
+		},
+	})
+	return framework.WaitForPodNameRunningInNamespace(f.ClientSet, p.name, f.Namespace.Name)
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/misc.go

@@ -0,0 +1,32 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// GetNodeIds returns the list of node names and panics in case of failure.
+func GetNodeIds(cs clientset.Interface) []string {
+	nodes := framework.GetReadySchedulableNodesOrDie(cs)
+	nodeIds := []string{}
+	for _, n := range nodes.Items {
+		nodeIds = append(nodeIds, n.Spec.ExternalID)
+	}
+	return nodeIds
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/types.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"regexp"
+	"strconv"
+
+	"sync"
+)
+
+var (
+	// Regexp, matching the contents of log entries, parsed or not
+	logEntryMessageRegex = regexp.MustCompile("(?:I\\d+ \\d+:\\d+:\\d+.\\d+       \\d+ logs_generator.go:67] )?(\\d+) .*")
+)
+
+// LogEntry represents a log entry, received from the logging backend.
+type LogEntry struct {
+	LogName     string
+	TextPayload string
+	JSONPayload map[string]interface{}
+}
+
+// TryGetEntryNumber returns the number of the log entry in sequence, if it
+// was generated by the load logging pod (requires special log format).
+func (entry LogEntry) TryGetEntryNumber() (int, bool) {
+	submatch := logEntryMessageRegex.FindStringSubmatch(entry.TextPayload)
+	if submatch == nil || len(submatch) < 2 {
+		return 0, false
+	}
+
+	lineNumber, err := strconv.Atoi(submatch[1])
+	return lineNumber, err == nil
+}
+
+// LogsQueueCollection is a thread-safe set of named log queues.
+type LogsQueueCollection interface {
+	Push(name string, logs ...LogEntry)
+	Pop(name string) []LogEntry
+}
+
+var _ LogsQueueCollection = &logsQueueCollection{}
+
+type logsQueueCollection struct {
+	mutex     *sync.Mutex
+	queues    map[string]chan LogEntry
+	queueSize int
+}
+
+// NewLogsQueueCollection returns a new LogsQueueCollection where each queue
+// is created with a default size of queueSize.
+func NewLogsQueueCollection(queueSize int) LogsQueueCollection {
+	return &logsQueueCollection{
+		mutex:     &sync.Mutex{},
+		queues:    map[string]chan LogEntry{},
+		queueSize: queueSize,
+	}
+}
+
+func (c *logsQueueCollection) Push(name string, logs ...LogEntry) {
+	q := c.getQueue(name)
+	for _, log := range logs {
+		q <- log
+	}
+}
+
+func (c *logsQueueCollection) Pop(name string) []LogEntry {
+	q := c.getQueue(name)
+	var entries []LogEntry
+polling_loop:
+	for {
+		select {
+		case entry := <-q:
+			entries = append(entries, entry)
+		default:
+			break polling_loop
+		}
+	}
+	return entries
+}
+
+func (c *logsQueueCollection) getQueue(name string) chan LogEntry {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	if q, ok := c.queues[name]; ok {
+		return q
+	}
+
+	newQ := make(chan LogEntry, c.queueSize)
+	c.queues[name] = newQ
+	return newQ
+}

vendor/k8s.io/kubernetes/test/e2e/instrumentation/logging/utils/wait.go

@@ -0,0 +1,217 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// LogChecker is an interface for an entity that can check whether logging
+// backend contains all wanted log entries.
+type LogChecker interface {
+	EntriesIngested() (bool, error)
+	Timeout() error
+}
+
+// IngestionPred is a type of a function that checks whether all required
+// log entries were ingested.
+type IngestionPred func(string, []LogEntry) (bool, error)
+
+// UntilFirstEntry is a IngestionPred that checks that at least one entry was
+// ingested.
+var UntilFirstEntry IngestionPred = func(_ string, entries []LogEntry) (bool, error) {
+	return len(entries) > 0, nil
+}
+
+// UntilFirstEntryFromLog is a IngestionPred that checks that at least one
+// entry from the log with a given name was ingested.
+func UntilFirstEntryFromLog(log string) IngestionPred {
+	return func(_ string, entries []LogEntry) (bool, error) {
+		for _, e := range entries {
+			if e.LogName == log {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+}
+
+// TimeoutFun is a function that is called when the waiting times out.
+type TimeoutFun func([]string, []bool) error
+
+// JustTimeout returns the error with the list of names for which backend is
+// still still missing logs.
+var JustTimeout TimeoutFun = func(names []string, ingested []bool) error {
+	failedNames := []string{}
+	for i, name := range names {
+		if !ingested[i] {
+			failedNames = append(failedNames, name)
+		}
+	}
+	return fmt.Errorf("timed out waiting for ingestion, still not ingested: %s",
+		strings.Join(failedNames, ","))
+}
+
+var _ LogChecker = &logChecker{}
+
+type logChecker struct {
+	provider      LogProvider
+	names         []string
+	ingested      []bool
+	ingestionPred IngestionPred
+	timeoutFun    TimeoutFun
+}
+
+// NewLogChecker constructs a LogChecker for a list of names from custom
+// IngestionPred and TimeoutFun.
+func NewLogChecker(p LogProvider, pred IngestionPred, timeout TimeoutFun, names ...string) LogChecker {
+	return &logChecker{
+		provider:      p,
+		names:         names,
+		ingested:      make([]bool, len(names)),
+		ingestionPred: pred,
+		timeoutFun:    timeout,
+	}
+}
+
+func (c *logChecker) EntriesIngested() (bool, error) {
+	allIngested := true
+	for i, name := range c.names {
+		if c.ingested[i] {
+			continue
+		}
+		entries := c.provider.ReadEntries(name)
+		ingested, err := c.ingestionPred(name, entries)
+		if err != nil {
+			return false, err
+		}
+		if ingested {
+			c.ingested[i] = true
+		}
+		allIngested = allIngested && ingested
+	}
+	return allIngested, nil
+}
+
+func (c *logChecker) Timeout() error {
+	return c.timeoutFun(c.names, c.ingested)
+}
+
+// NumberedIngestionPred is a IngestionPred that takes into account sequential
+// numbers of ingested entries.
+type NumberedIngestionPred func(string, map[int]bool) (bool, error)
+
+// NumberedTimeoutFun is a TimeoutFun that takes into account sequential
+// numbers of ingested entries.
+type NumberedTimeoutFun func([]string, map[string]map[int]bool) error
+
+// NewNumberedLogChecker returns a log checker that works with numbered log
+// entries generated by load logging pods.
+func NewNumberedLogChecker(p LogProvider, pred NumberedIngestionPred,
+	timeout NumberedTimeoutFun, names ...string) LogChecker {
+	occs := map[string]map[int]bool{}
+	return NewLogChecker(p, func(name string, entries []LogEntry) (bool, error) {
+		occ, ok := occs[name]
+		if !ok {
+			occ = map[int]bool{}
+			occs[name] = occ
+		}
+		for _, entry := range entries {
+			if no, ok := entry.TryGetEntryNumber(); ok {
+				occ[no] = true
+			}
+		}
+		return pred(name, occ)
+	}, func(names []string, _ []bool) error {
+		return timeout(names, occs)
+	}, names...)
+}
+
+// NewFullIngestionPodLogChecker returns a log checks that works with numbered
+// log entries generated by load logging pods and waits until all entries are
+// ingested. If timeout is reached, fraction is lost logs up to slack is
+// considered tolerable.
+func NewFullIngestionPodLogChecker(p LogProvider, slack float64, pods ...FiniteLoggingPod) LogChecker {
+	podsMap := map[string]FiniteLoggingPod{}
+	for _, p := range pods {
+		podsMap[p.Name()] = p
+	}
+	return NewNumberedLogChecker(p, getFullIngestionPred(podsMap),
+		getFullIngestionTimeout(podsMap, slack), getFiniteLoggingPodNames(pods)...)
+}
+
+func getFullIngestionPred(podsMap map[string]FiniteLoggingPod) NumberedIngestionPred {
+	return func(name string, occ map[int]bool) (bool, error) {
+		p := podsMap[name]
+		ok := len(occ) == p.ExpectedLineCount()
+		return ok, nil
+	}
+}
+
+func getFullIngestionTimeout(podsMap map[string]FiniteLoggingPod, slack float64) NumberedTimeoutFun {
+	return func(names []string, occs map[string]map[int]bool) error {
+		totalGot, totalWant := 0, 0
+		lossMsgs := []string{}
+		for _, name := range names {
+			got := len(occs[name])
+			want := podsMap[name].ExpectedLineCount()
+			if got != want {
+				lossMsg := fmt.Sprintf("%s: %d lines", name, want-got)
+				lossMsgs = append(lossMsgs, lossMsg)
+			}
+			totalGot += got
+			totalWant += want
+		}
+		if len(lossMsgs) > 0 {
+			framework.Logf("Still missing logs from:\n%s", strings.Join(lossMsgs, "\n"))
+		}
+		lostFrac := 1 - float64(totalGot)/float64(totalWant)
+		if lostFrac > slack {
+			return fmt.Errorf("still missing %.2f%% of logs, only %.2f%% is tolerable",
+				lostFrac*100, slack*100)
+		}
+		framework.Logf("Missing %.2f%% of logs, which is lower than the threshold %.2f%%",
+			lostFrac*100, slack*100)
+		return nil
+	}
+}
+
+// WaitForLogs checks that logs are ingested, as reported by the log checker
+// until the timeout has passed. Function sleeps for interval between two
+// log ingestion checks.
+func WaitForLogs(c LogChecker, interval, timeout time.Duration) error {
+	err := wait.Poll(interval, timeout, func() (bool, error) {
+		return c.EntriesIngested()
+	})
+	if err == wait.ErrWaitTimeout {
+		return c.Timeout()
+	}
+	return err
+}
+
+func getFiniteLoggingPodNames(pods []FiniteLoggingPod) []string {
+	names := []string{}
+	for _, p := range pods {
+		names = append(names, p.Name())
+	}
+	return names
+}