vendor files

vendor/k8s.io/kubernetes/pkg/proxy/healthcheck/BUILD

@@ -0,0 +1,52 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "healthcheck.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/proxy/healthcheck",
+    deps = [
+        "//pkg/apis/core:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/renstrom/dedent:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["healthcheck_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/proxy/healthcheck",
+    library = ":go_default_library",
+    deps = [
+        "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/proxy/healthcheck/OWNERS

@@ -0,0 +1,2 @@
+reviewers:
+- m1093782566

vendor/k8s.io/kubernetes/pkg/proxy/healthcheck/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package healthcheck provides tools for serving kube-proxy healthchecks.
+package healthcheck // import "k8s.io/kubernetes/pkg/proxy/healthcheck"

vendor/k8s.io/kubernetes/pkg/proxy/healthcheck/healthcheck.go

@@ -0,0 +1,347 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package healthcheck
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/renstrom/dedent"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/record"
+	api "k8s.io/kubernetes/pkg/apis/core"
+)
+
+var nodeHealthzRetryInterval = 60 * time.Second
+
+// Server serves HTTP endpoints for each service name, with results
+// based on the endpoints.  If there are 0 endpoints for a service, it returns a
+// 503 "Service Unavailable" error (telling LBs not to use this node).  If there
+// are 1 or more endpoints, it returns a 200 "OK".
+type Server interface {
+	// Make the new set of services be active.  Services that were open before
+	// will be closed.  Services that are new will be opened.  Service that
+	// existed and are in the new set will be left alone.  The value of the map
+	// is the healthcheck-port to listen on.
+	SyncServices(newServices map[types.NamespacedName]uint16) error
+	// Make the new set of endpoints be active.  Endpoints for services that do
+	// not exist will be dropped.  The value of the map is the number of
+	// endpoints the service has on this node.
+	SyncEndpoints(newEndpoints map[types.NamespacedName]int) error
+}
+
+// Listener allows for testing of Server.  If the Listener argument
+// to NewServer() is nil, the real net.Listen function will be used.
+type Listener interface {
+	// Listen is very much like net.Listen, except the first arg (network) is
+	// fixed to be "tcp".
+	Listen(addr string) (net.Listener, error)
+}
+
+// HTTPServerFactory allows for testing of Server.  If the
+// HTTPServerFactory argument to NewServer() is nil, the real
+// http.Server type will be used.
+type HTTPServerFactory interface {
+	// New creates an instance of a type satisfying HTTPServer.  This is
+	// designed to include http.Server.
+	New(addr string, handler http.Handler) HTTPServer
+}
+
+// HTTPServer allows for testing of Server.
+type HTTPServer interface {
+	// Server is designed so that http.Server satifies this interface,
+	Serve(listener net.Listener) error
+}
+
+// NewServer allocates a new healthcheck server manager.  If either
+// of the injected arguments are nil, defaults will be used.
+func NewServer(hostname string, recorder record.EventRecorder, listener Listener, httpServerFactory HTTPServerFactory) Server {
+	if listener == nil {
+		listener = stdNetListener{}
+	}
+	if httpServerFactory == nil {
+		httpServerFactory = stdHTTPServerFactory{}
+	}
+	return &server{
+		hostname:    hostname,
+		recorder:    recorder,
+		listener:    listener,
+		httpFactory: httpServerFactory,
+		services:    map[types.NamespacedName]*hcInstance{},
+	}
+}
+
+// Implement Listener in terms of net.Listen.
+type stdNetListener struct{}
+
+func (stdNetListener) Listen(addr string) (net.Listener, error) {
+	return net.Listen("tcp", addr)
+}
+
+var _ Listener = stdNetListener{}
+
+// Implement HTTPServerFactory in terms of http.Server.
+type stdHTTPServerFactory struct{}
+
+func (stdHTTPServerFactory) New(addr string, handler http.Handler) HTTPServer {
+	return &http.Server{
+		Addr:    addr,
+		Handler: handler,
+	}
+}
+
+var _ HTTPServerFactory = stdHTTPServerFactory{}
+
+type server struct {
+	hostname    string
+	recorder    record.EventRecorder // can be nil
+	listener    Listener
+	httpFactory HTTPServerFactory
+
+	lock     sync.Mutex
+	services map[types.NamespacedName]*hcInstance
+}
+
+func (hcs *server) SyncServices(newServices map[types.NamespacedName]uint16) error {
+	hcs.lock.Lock()
+	defer hcs.lock.Unlock()
+
+	// Remove any that are not needed any more.
+	for nsn, svc := range hcs.services {
+		if port, found := newServices[nsn]; !found || port != svc.port {
+			glog.V(2).Infof("Closing healthcheck %q on port %d", nsn.String(), svc.port)
+			if err := svc.listener.Close(); err != nil {
+				glog.Errorf("Close(%v): %v", svc.listener.Addr(), err)
+			}
+			delete(hcs.services, nsn)
+		}
+	}
+
+	// Add any that are needed.
+	for nsn, port := range newServices {
+		if hcs.services[nsn] != nil {
+			glog.V(3).Infof("Existing healthcheck %q on port %d", nsn.String(), port)
+			continue
+		}
+
+		glog.V(2).Infof("Opening healthcheck %q on port %d", nsn.String(), port)
+		svc := &hcInstance{port: port}
+		addr := fmt.Sprintf(":%d", port)
+		svc.server = hcs.httpFactory.New(addr, hcHandler{name: nsn, hcs: hcs})
+		var err error
+		svc.listener, err = hcs.listener.Listen(addr)
+		if err != nil {
+			msg := fmt.Sprintf("node %s failed to start healthcheck %q on port %d: %v", hcs.hostname, nsn.String(), port, err)
+
+			if hcs.recorder != nil {
+				hcs.recorder.Eventf(
+					&v1.ObjectReference{
+						Kind:      "Service",
+						Namespace: nsn.Namespace,
+						Name:      nsn.Name,
+						UID:       types.UID(nsn.String()),
+					}, api.EventTypeWarning, "FailedToStartServiceHealthcheck", msg)
+			}
+			glog.Error(msg)
+			continue
+		}
+		hcs.services[nsn] = svc
+
+		go func(nsn types.NamespacedName, svc *hcInstance) {
+			// Serve() will exit when the listener is closed.
+			glog.V(3).Infof("Starting goroutine for healthcheck %q on port %d", nsn.String(), svc.port)
+			if err := svc.server.Serve(svc.listener); err != nil {
+				glog.V(3).Infof("Healthcheck %q closed: %v", nsn.String(), err)
+				return
+			}
+			glog.V(3).Infof("Healthcheck %q closed", nsn.String())
+		}(nsn, svc)
+	}
+	return nil
+}
+
+type hcInstance struct {
+	port      uint16
+	listener  net.Listener
+	server    HTTPServer
+	endpoints int // number of local endpoints for a service
+}
+
+type hcHandler struct {
+	name types.NamespacedName
+	hcs  *server
+}
+
+var _ http.Handler = hcHandler{}
+
+func (h hcHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	h.hcs.lock.Lock()
+	svc, ok := h.hcs.services[h.name]
+	if !ok || svc == nil {
+		h.hcs.lock.Unlock()
+		glog.Errorf("Received request for closed healthcheck %q", h.name.String())
+		return
+	}
+	count := svc.endpoints
+	h.hcs.lock.Unlock()
+
+	resp.Header().Set("Content-Type", "application/json")
+	if count == 0 {
+		resp.WriteHeader(http.StatusServiceUnavailable)
+	} else {
+		resp.WriteHeader(http.StatusOK)
+	}
+	fmt.Fprintf(resp, strings.Trim(dedent.Dedent(fmt.Sprintf(`
+		{
+			"service": {
+				"namespace": %q,
+				"name": %q
+			},
+			"localEndpoints": %d
+		}
+		`, h.name.Namespace, h.name.Name, count)), "\n"))
+}
+
+func (hcs *server) SyncEndpoints(newEndpoints map[types.NamespacedName]int) error {
+	hcs.lock.Lock()
+	defer hcs.lock.Unlock()
+
+	for nsn, count := range newEndpoints {
+		if hcs.services[nsn] == nil {
+			glog.V(3).Infof("Not saving endpoints for unknown healthcheck %q", nsn.String())
+			continue
+		}
+		glog.V(3).Infof("Reporting %d endpoints for healthcheck %q", count, nsn.String())
+		hcs.services[nsn].endpoints = count
+	}
+	for nsn, hci := range hcs.services {
+		if _, found := newEndpoints[nsn]; !found {
+			hci.endpoints = 0
+		}
+	}
+	return nil
+}
+
+// HealthzUpdater allows callers to update healthz timestamp only.
+type HealthzUpdater interface {
+	UpdateTimestamp()
+}
+
+// HealthzServer returns 200 "OK" by default. Once timestamp has been
+// updated, it verifies we don't exceed max no respond duration since
+// last update.
+type HealthzServer struct {
+	listener    Listener
+	httpFactory HTTPServerFactory
+	clock       clock.Clock
+
+	addr          string
+	port          int32
+	healthTimeout time.Duration
+	recorder      record.EventRecorder
+	nodeRef       *v1.ObjectReference
+
+	lastUpdated atomic.Value
+}
+
+// NewDefaultHealthzServer returns a default healthz http server.
+func NewDefaultHealthzServer(addr string, healthTimeout time.Duration, recorder record.EventRecorder, nodeRef *v1.ObjectReference) *HealthzServer {
+	return newHealthzServer(nil, nil, nil, addr, healthTimeout, recorder, nodeRef)
+}
+
+func newHealthzServer(listener Listener, httpServerFactory HTTPServerFactory, c clock.Clock, addr string, healthTimeout time.Duration, recorder record.EventRecorder, nodeRef *v1.ObjectReference) *HealthzServer {
+	if listener == nil {
+		listener = stdNetListener{}
+	}
+	if httpServerFactory == nil {
+		httpServerFactory = stdHTTPServerFactory{}
+	}
+	if c == nil {
+		c = clock.RealClock{}
+	}
+	return &HealthzServer{
+		listener:      listener,
+		httpFactory:   httpServerFactory,
+		clock:         c,
+		addr:          addr,
+		healthTimeout: healthTimeout,
+		recorder:      recorder,
+		nodeRef:       nodeRef,
+	}
+}
+
+// UpdateTimestamp updates the lastUpdated timestamp.
+func (hs *HealthzServer) UpdateTimestamp() {
+	hs.lastUpdated.Store(hs.clock.Now())
+}
+
+// Run starts the healthz http server and returns.
+func (hs *HealthzServer) Run() {
+	serveMux := http.NewServeMux()
+	serveMux.Handle("/healthz", healthzHandler{hs: hs})
+	server := hs.httpFactory.New(hs.addr, serveMux)
+
+	go wait.Until(func() {
+		glog.V(3).Infof("Starting goroutine for healthz on %s", hs.addr)
+
+		listener, err := hs.listener.Listen(hs.addr)
+		if err != nil {
+			msg := fmt.Sprintf("Failed to start node healthz on %s: %v", hs.addr, err)
+			if hs.recorder != nil {
+				hs.recorder.Eventf(hs.nodeRef, api.EventTypeWarning, "FailedToStartNodeHealthcheck", msg)
+			}
+			glog.Error(msg)
+			return
+		}
+
+		if err := server.Serve(listener); err != nil {
+			glog.Errorf("Healthz closed with error: %v", err)
+			return
+		}
+		glog.Errorf("Unexpected healthz closed.")
+	}, nodeHealthzRetryInterval, wait.NeverStop)
+}
+
+type healthzHandler struct {
+	hs *HealthzServer
+}
+
+func (h healthzHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+	lastUpdated := time.Time{}
+	if val := h.hs.lastUpdated.Load(); val != nil {
+		lastUpdated = val.(time.Time)
+	}
+	currentTime := h.hs.clock.Now()
+
+	resp.Header().Set("Content-Type", "application/json")
+	if !lastUpdated.IsZero() && currentTime.After(lastUpdated.Add(h.hs.healthTimeout)) {
+		resp.WriteHeader(http.StatusServiceUnavailable)
+	} else {
+		resp.WriteHeader(http.StatusOK)
+	}
+	fmt.Fprintf(resp, fmt.Sprintf(`{"lastUpdated": %q,"currentTime": %q}`, lastUpdated, currentTime))
+}

vendor/k8s.io/kubernetes/pkg/proxy/healthcheck/healthcheck_test.go

@@ -0,0 +1,405 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package healthcheck
+
+import (
+	"encoding/json"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/sets"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+type fakeListener struct {
+	openPorts sets.String
+}
+
+func newFakeListener() *fakeListener {
+	return &fakeListener{
+		openPorts: sets.String{},
+	}
+}
+
+func (fake *fakeListener) hasPort(addr string) bool {
+	return fake.openPorts.Has(addr)
+}
+
+func (fake *fakeListener) Listen(addr string) (net.Listener, error) {
+	fake.openPorts.Insert(addr)
+	return &fakeNetListener{
+		parent: fake,
+		addr:   addr,
+	}, nil
+}
+
+type fakeNetListener struct {
+	parent *fakeListener
+	addr   string
+}
+
+func (fake *fakeNetListener) Accept() (net.Conn, error) {
+	// Not implemented
+	return nil, nil
+}
+
+func (fake *fakeNetListener) Close() error {
+	fake.parent.openPorts.Delete(fake.addr)
+	return nil
+}
+
+func (fake *fakeNetListener) Addr() net.Addr {
+	// Not implemented
+	return nil
+}
+
+type fakeHTTPServerFactory struct{}
+
+func newFakeHTTPServerFactory() *fakeHTTPServerFactory {
+	return &fakeHTTPServerFactory{}
+}
+
+func (fake *fakeHTTPServerFactory) New(addr string, handler http.Handler) HTTPServer {
+	return &fakeHTTPServer{
+		addr:    addr,
+		handler: handler,
+	}
+}
+
+type fakeHTTPServer struct {
+	addr    string
+	handler http.Handler
+}
+
+func (fake *fakeHTTPServer) Serve(listener net.Listener) error {
+	return nil // Cause the goroutine to return
+}
+
+func mknsn(ns, name string) types.NamespacedName {
+	return types.NamespacedName{
+		Namespace: ns,
+		Name:      name,
+	}
+}
+
+type hcPayload struct {
+	Service struct {
+		Namespace string
+		Name      string
+	}
+	LocalEndpoints int
+}
+
+type healthzPayload struct {
+	LastUpdated string
+	CurrentTime string
+}
+
+func TestServer(t *testing.T) {
+	listener := newFakeListener()
+	httpFactory := newFakeHTTPServerFactory()
+
+	hcsi := NewServer("hostname", nil, listener, httpFactory)
+	hcs := hcsi.(*server)
+	if len(hcs.services) != 0 {
+		t.Errorf("expected 0 services, got %d", len(hcs.services))
+	}
+
+	// sync nothing
+	hcs.SyncServices(nil)
+	if len(hcs.services) != 0 {
+		t.Errorf("expected 0 services, got %d", len(hcs.services))
+	}
+	hcs.SyncEndpoints(nil)
+	if len(hcs.services) != 0 {
+		t.Errorf("expected 0 services, got %d", len(hcs.services))
+	}
+
+	// sync unknown endpoints, should be dropped
+	hcs.SyncEndpoints(map[types.NamespacedName]int{mknsn("a", "b"): 93})
+	if len(hcs.services) != 0 {
+		t.Errorf("expected 0 services, got %d", len(hcs.services))
+	}
+
+	// sync a real service
+	nsn := mknsn("a", "b")
+	hcs.SyncServices(map[types.NamespacedName]uint16{nsn: 9376})
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	if len(listener.openPorts) != 1 {
+		t.Errorf("expected 1 open port, got %d\n%s", len(listener.openPorts), spew.Sdump(listener.openPorts))
+	}
+	if !listener.hasPort(":9376") {
+		t.Errorf("expected port :9376 to be open\n%s", spew.Sdump(listener.openPorts))
+	}
+	// test the handler
+	testHandler(hcs, nsn, http.StatusServiceUnavailable, 0, t)
+
+	// sync an endpoint
+	hcs.SyncEndpoints(map[types.NamespacedName]int{nsn: 18})
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 18 {
+		t.Errorf("expected 18 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	// test the handler
+	testHandler(hcs, nsn, http.StatusOK, 18, t)
+
+	// sync zero endpoints
+	hcs.SyncEndpoints(map[types.NamespacedName]int{nsn: 0})
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	// test the handler
+	testHandler(hcs, nsn, http.StatusServiceUnavailable, 0, t)
+
+	// put the endpoint back
+	hcs.SyncEndpoints(map[types.NamespacedName]int{nsn: 11})
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 11 {
+		t.Errorf("expected 18 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	// sync nil endpoints
+	hcs.SyncEndpoints(nil)
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	// test the handler
+	testHandler(hcs, nsn, http.StatusServiceUnavailable, 0, t)
+
+	// put the endpoint back
+	hcs.SyncEndpoints(map[types.NamespacedName]int{nsn: 18})
+	if len(hcs.services) != 1 {
+		t.Errorf("expected 1 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn].endpoints != 18 {
+		t.Errorf("expected 18 endpoints, got %d", hcs.services[nsn].endpoints)
+	}
+	// delete the service
+	hcs.SyncServices(nil)
+	if len(hcs.services) != 0 {
+		t.Errorf("expected 0 services, got %d", len(hcs.services))
+	}
+
+	// sync multiple services
+	nsn1 := mknsn("a", "b")
+	nsn2 := mknsn("c", "d")
+	nsn3 := mknsn("e", "f")
+	nsn4 := mknsn("g", "h")
+	hcs.SyncServices(map[types.NamespacedName]uint16{
+		nsn1: 9376,
+		nsn2: 12909,
+		nsn3: 11113,
+	})
+	if len(hcs.services) != 3 {
+		t.Errorf("expected 3 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn1].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn1].endpoints)
+	}
+	if hcs.services[nsn2].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn2].endpoints)
+	}
+	if hcs.services[nsn3].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn3].endpoints)
+	}
+	if len(listener.openPorts) != 3 {
+		t.Errorf("expected 3 open ports, got %d\n%s", len(listener.openPorts), spew.Sdump(listener.openPorts))
+	}
+	// test the handlers
+	testHandler(hcs, nsn1, http.StatusServiceUnavailable, 0, t)
+	testHandler(hcs, nsn2, http.StatusServiceUnavailable, 0, t)
+	testHandler(hcs, nsn3, http.StatusServiceUnavailable, 0, t)
+
+	// sync endpoints
+	hcs.SyncEndpoints(map[types.NamespacedName]int{
+		nsn1: 9,
+		nsn2: 3,
+		nsn3: 7,
+	})
+	if len(hcs.services) != 3 {
+		t.Errorf("expected 3 services, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn1].endpoints != 9 {
+		t.Errorf("expected 9 endpoints, got %d", hcs.services[nsn1].endpoints)
+	}
+	if hcs.services[nsn2].endpoints != 3 {
+		t.Errorf("expected 3 endpoints, got %d", hcs.services[nsn2].endpoints)
+	}
+	if hcs.services[nsn3].endpoints != 7 {
+		t.Errorf("expected 7 endpoints, got %d", hcs.services[nsn3].endpoints)
+	}
+	// test the handlers
+	testHandler(hcs, nsn1, http.StatusOK, 9, t)
+	testHandler(hcs, nsn2, http.StatusOK, 3, t)
+	testHandler(hcs, nsn3, http.StatusOK, 7, t)
+
+	// sync new services
+	hcs.SyncServices(map[types.NamespacedName]uint16{
+		//nsn1: 9376, // remove it
+		nsn2: 12909, // leave it
+		nsn3: 11114, // change it
+		nsn4: 11878, // add it
+	})
+	if len(hcs.services) != 3 {
+		t.Errorf("expected 3 service, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn2].endpoints != 3 {
+		t.Errorf("expected 3 endpoints, got %d", hcs.services[nsn2].endpoints)
+	}
+	if hcs.services[nsn3].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn3].endpoints)
+	}
+	if hcs.services[nsn4].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn4].endpoints)
+	}
+	// test the handlers
+	testHandler(hcs, nsn2, http.StatusOK, 3, t)
+	testHandler(hcs, nsn3, http.StatusServiceUnavailable, 0, t)
+	testHandler(hcs, nsn4, http.StatusServiceUnavailable, 0, t)
+
+	// sync endpoints
+	hcs.SyncEndpoints(map[types.NamespacedName]int{
+		nsn1: 9,
+		nsn2: 3,
+		nsn3: 7,
+		nsn4: 6,
+	})
+	if len(hcs.services) != 3 {
+		t.Errorf("expected 3 services, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn2].endpoints != 3 {
+		t.Errorf("expected 3 endpoints, got %d", hcs.services[nsn2].endpoints)
+	}
+	if hcs.services[nsn3].endpoints != 7 {
+		t.Errorf("expected 7 endpoints, got %d", hcs.services[nsn3].endpoints)
+	}
+	if hcs.services[nsn4].endpoints != 6 {
+		t.Errorf("expected 6 endpoints, got %d", hcs.services[nsn4].endpoints)
+	}
+	// test the handlers
+	testHandler(hcs, nsn2, http.StatusOK, 3, t)
+	testHandler(hcs, nsn3, http.StatusOK, 7, t)
+	testHandler(hcs, nsn4, http.StatusOK, 6, t)
+
+	// sync endpoints, missing nsn2
+	hcs.SyncEndpoints(map[types.NamespacedName]int{
+		nsn3: 7,
+		nsn4: 6,
+	})
+	if len(hcs.services) != 3 {
+		t.Errorf("expected 3 services, got %d", len(hcs.services))
+	}
+	if hcs.services[nsn2].endpoints != 0 {
+		t.Errorf("expected 0 endpoints, got %d", hcs.services[nsn2].endpoints)
+	}
+	if hcs.services[nsn3].endpoints != 7 {
+		t.Errorf("expected 7 endpoints, got %d", hcs.services[nsn3].endpoints)
+	}
+	if hcs.services[nsn4].endpoints != 6 {
+		t.Errorf("expected 6 endpoints, got %d", hcs.services[nsn4].endpoints)
+	}
+	// test the handlers
+	testHandler(hcs, nsn2, http.StatusServiceUnavailable, 0, t)
+	testHandler(hcs, nsn3, http.StatusOK, 7, t)
+	testHandler(hcs, nsn4, http.StatusOK, 6, t)
+}
+
+func testHandler(hcs *server, nsn types.NamespacedName, status int, endpoints int, t *testing.T) {
+	handler := hcs.services[nsn].server.(*fakeHTTPServer).handler
+	req, err := http.NewRequest("GET", "/healthz", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp := httptest.NewRecorder()
+
+	handler.ServeHTTP(resp, req)
+
+	if resp.Code != status {
+		t.Errorf("expected status code %v, got %v", status, resp.Code)
+	}
+	var payload hcPayload
+	if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil {
+		t.Fatal(err)
+	}
+	if payload.Service.Name != nsn.Name || payload.Service.Namespace != nsn.Namespace {
+		t.Errorf("expected payload name %q, got %v", nsn.String(), payload.Service)
+	}
+	if payload.LocalEndpoints != endpoints {
+		t.Errorf("expected %d endpoints, got %d", endpoints, payload.LocalEndpoints)
+	}
+}
+
+func TestHealthzServer(t *testing.T) {
+	listener := newFakeListener()
+	httpFactory := newFakeHTTPServerFactory()
+	fakeClock := clock.NewFakeClock(time.Now())
+
+	hs := newHealthzServer(listener, httpFactory, fakeClock, "127.0.0.1:10256", 10*time.Second, nil, nil)
+	server := hs.httpFactory.New(hs.addr, healthzHandler{hs: hs})
+
+	// Should return 200 "OK" by default.
+	testHealthzHandler(server, http.StatusOK, t)
+
+	// Should return 503 "ServiceUnavailable" if exceed max no respond duration.
+	hs.UpdateTimestamp()
+	fakeClock.Step(25 * time.Second)
+	testHealthzHandler(server, http.StatusServiceUnavailable, t)
+
+	// Should return 200 "OK" if timestamp is valid.
+	hs.UpdateTimestamp()
+	fakeClock.Step(5 * time.Second)
+	testHealthzHandler(server, http.StatusOK, t)
+}
+
+func testHealthzHandler(server HTTPServer, status int, t *testing.T) {
+	handler := server.(*fakeHTTPServer).handler
+	req, err := http.NewRequest("GET", "/healthz", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp := httptest.NewRecorder()
+
+	handler.ServeHTTP(resp, req)
+
+	if resp.Code != status {
+		t.Errorf("expected status code %v, got %v", status, resp.Code)
+	}
+	var payload healthzPayload
+	if err := json.Unmarshal(resp.Body.Bytes(), &payload); err != nil {
+		t.Fatal(err)
+	}
+}