vendor files

vendor/k8s.io/kubernetes/pkg/kubelet/network/BUILD

@@ -0,0 +1,52 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "network.go",
+        "plugins.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network",
+    deps = [
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/network/hostport:go_default_library",
+        "//pkg/kubelet/network/metrics:go_default_library",
+        "//pkg/util/sysctl:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/validation:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/network/cni:all-srcs",
+        "//pkg/kubelet/network/dns:all-srcs",
+        "//pkg/kubelet/network/hairpin:all-srcs",
+        "//pkg/kubelet/network/hostport:all-srcs",
+        "//pkg/kubelet/network/kubenet:all-srcs",
+        "//pkg/kubelet/network/metrics:all-srcs",
+        "//pkg/kubelet/network/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/OWNERS

@@ -0,0 +1,8 @@
+approvers:
+- thockin
+- dchen1107
+- matchstick
+- freehan
+reviewers:
+- sig-network-reviewers
+

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/BUILD

@@ -0,0 +1,82 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cni.go",
+        "cni_others.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "cni_windows.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/cni",
+    deps = [
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/network:go_default_library",
+        "//vendor/github.com/containernetworking/cni/libcni:go_default_library",
+        "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:windows_amd64": [
+            "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "cni_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/cni",
+    library = ":go_default_library",
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+            "//pkg/kubelet/container:go_default_library",
+            "//pkg/kubelet/container/testing:go_default_library",
+            "//pkg/kubelet/network:go_default_library",
+            "//pkg/kubelet/network/cni/testing:go_default_library",
+            "//pkg/kubelet/network/hostport:go_default_library",
+            "//pkg/kubelet/network/testing:go_default_library",
+            "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
+            "//vendor/github.com/stretchr/testify/mock:go_default_library",
+            "//vendor/k8s.io/api/core/v1:go_default_library",
+            "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+            "//vendor/k8s.io/client-go/util/testing:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/network/cni/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/cni.go

@@ -0,0 +1,322 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/containernetworking/cni/libcni"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	utilexec "k8s.io/utils/exec"
+)
+
+const (
+	CNIPluginName        = "cni"
+	DefaultNetDir        = "/etc/cni/net.d"
+	DefaultCNIDir        = "/opt/cni/bin"
+	VendorCNIDirTemplate = "%s/opt/%s/bin"
+)
+
+type cniNetworkPlugin struct {
+	network.NoopNetworkPlugin
+
+	loNetwork *cniNetwork
+
+	sync.RWMutex
+	defaultNetwork *cniNetwork
+
+	host               network.Host
+	execer             utilexec.Interface
+	nsenterPath        string
+	pluginDir          string
+	binDir             string
+	vendorCNIDirPrefix string
+}
+
+type cniNetwork struct {
+	name          string
+	NetworkConfig *libcni.NetworkConfigList
+	CNIConfig     libcni.CNI
+}
+
+// cniPortMapping maps to the standard CNI portmapping Capability
+// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md
+type cniPortMapping struct {
+	HostPort      int32  `json:"hostPort"`
+	ContainerPort int32  `json:"containerPort"`
+	Protocol      string `json:"protocol"`
+	HostIP        string `json:"hostIP"`
+}
+
+func probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, vendorCNIDirPrefix string) []network.NetworkPlugin {
+	if binDir == "" {
+		binDir = DefaultCNIDir
+	}
+	plugin := &cniNetworkPlugin{
+		defaultNetwork:     nil,
+		loNetwork:          getLoNetwork(binDir, vendorCNIDirPrefix),
+		execer:             utilexec.New(),
+		pluginDir:          pluginDir,
+		binDir:             binDir,
+		vendorCNIDirPrefix: vendorCNIDirPrefix,
+	}
+
+	// sync NetworkConfig in best effort during probing.
+	plugin.syncNetworkConfig()
+	return []network.NetworkPlugin{plugin}
+}
+
+func ProbeNetworkPlugins(pluginDir, binDir string) []network.NetworkPlugin {
+	return probeNetworkPluginsWithVendorCNIDirPrefix(pluginDir, binDir, "")
+}
+
+func getDefaultCNINetwork(pluginDir, binDir, vendorCNIDirPrefix string) (*cniNetwork, error) {
+	if pluginDir == "" {
+		pluginDir = DefaultNetDir
+	}
+	files, err := libcni.ConfFiles(pluginDir, []string{".conf", ".conflist", ".json"})
+	switch {
+	case err != nil:
+		return nil, err
+	case len(files) == 0:
+		return nil, fmt.Errorf("No networks found in %s", pluginDir)
+	}
+
+	sort.Strings(files)
+	for _, confFile := range files {
+		var confList *libcni.NetworkConfigList
+		if strings.HasSuffix(confFile, ".conflist") {
+			confList, err = libcni.ConfListFromFile(confFile)
+			if err != nil {
+				glog.Warningf("Error loading CNI config list file %s: %v", confFile, err)
+				continue
+			}
+		} else {
+			conf, err := libcni.ConfFromFile(confFile)
+			if err != nil {
+				glog.Warningf("Error loading CNI config file %s: %v", confFile, err)
+				continue
+			}
+			// Ensure the config has a "type" so we know what plugin to run.
+			// Also catches the case where somebody put a conflist into a conf file.
+			if conf.Network.Type == "" {
+				glog.Warningf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
+				continue
+			}
+
+			confList, err = libcni.ConfListFromConf(conf)
+			if err != nil {
+				glog.Warningf("Error converting CNI config file %s to list: %v", confFile, err)
+				continue
+			}
+		}
+		if len(confList.Plugins) == 0 {
+			glog.Warningf("CNI config list %s has no networks, skipping", confFile)
+			continue
+		}
+		confType := confList.Plugins[0].Network.Type
+
+		// Search for vendor-specific plugins as well as default plugins in the CNI codebase.
+		vendorDir := vendorCNIDir(vendorCNIDirPrefix, confType)
+		cninet := &libcni.CNIConfig{
+			Path: []string{vendorDir, binDir},
+		}
+		network := &cniNetwork{name: confList.Name, NetworkConfig: confList, CNIConfig: cninet}
+		return network, nil
+	}
+	return nil, fmt.Errorf("No valid networks found in %s", pluginDir)
+}
+
+func vendorCNIDir(prefix, pluginType string) string {
+	return fmt.Sprintf(VendorCNIDirTemplate, prefix, pluginType)
+}
+
+func (plugin *cniNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	err := plugin.platformInit()
+	if err != nil {
+		return err
+	}
+
+	plugin.host = host
+
+	plugin.syncNetworkConfig()
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) syncNetworkConfig() {
+	network, err := getDefaultCNINetwork(plugin.pluginDir, plugin.binDir, plugin.vendorCNIDirPrefix)
+	if err != nil {
+		glog.Warningf("Unable to update cni config: %s", err)
+		return
+	}
+	plugin.setDefaultNetwork(network)
+}
+
+func (plugin *cniNetworkPlugin) getDefaultNetwork() *cniNetwork {
+	plugin.RLock()
+	defer plugin.RUnlock()
+	return plugin.defaultNetwork
+}
+
+func (plugin *cniNetworkPlugin) setDefaultNetwork(n *cniNetwork) {
+	plugin.Lock()
+	defer plugin.Unlock()
+	plugin.defaultNetwork = n
+}
+
+func (plugin *cniNetworkPlugin) checkInitialized() error {
+	if plugin.getDefaultNetwork() == nil {
+		return errors.New("cni config uninitialized")
+	}
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) Name() string {
+	return CNIPluginName
+}
+
+func (plugin *cniNetworkPlugin) Status() error {
+	// sync network config from pluginDir periodically to detect network config updates
+	plugin.syncNetworkConfig()
+
+	// Can't set up pods if we don't have any CNI network configs yet
+	return plugin.checkInitialized()
+}
+
+func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	if err := plugin.checkInitialized(); err != nil {
+		return err
+	}
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+	}
+
+	// Windows doesn't have loNetwork. It comes only with Linux
+	if plugin.loNetwork != nil {
+		if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath); err != nil {
+			glog.Errorf("Error while adding to cni lo network: %s", err)
+			return err
+		}
+	}
+
+	_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
+	if err != nil {
+		glog.Errorf("Error while adding to cni network: %s", err)
+		return err
+	}
+
+	return err
+}
+
+func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.ContainerID) error {
+	if err := plugin.checkInitialized(); err != nil {
+		return err
+	}
+
+	// Lack of namespace should not be fatal on teardown
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		glog.Warningf("CNI failed to retrieve network namespace path: %v", err)
+	}
+
+	return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
+}
+
+func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) {
+	rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
+	if err != nil {
+		glog.Errorf("Error adding network when building cni runtime conf: %v", err)
+		return nil, err
+	}
+
+	netConf, cniNet := network.NetworkConfig, network.CNIConfig
+	glog.V(4).Infof("About to add CNI network %v (type=%v)", netConf.Name, netConf.Plugins[0].Network.Type)
+	res, err := cniNet.AddNetworkList(netConf, rt)
+	if err != nil {
+		glog.Errorf("Error adding network: %v", err)
+		return nil, err
+	}
+
+	return res, nil
+}
+
+func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) error {
+	rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
+	if err != nil {
+		glog.Errorf("Error deleting network when building cni runtime conf: %v", err)
+		return err
+	}
+
+	netConf, cniNet := network.NetworkConfig, network.CNIConfig
+	glog.V(4).Infof("About to del CNI network %v (type=%v)", netConf.Name, netConf.Plugins[0].Network.Type)
+	err = cniNet.DelNetworkList(netConf, rt)
+	if err != nil {
+		glog.Errorf("Error deleting network: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (*libcni.RuntimeConf, error) {
+	glog.V(4).Infof("Got netns path %v", podNetnsPath)
+	glog.V(4).Infof("Using podns path %v", podNs)
+
+	rt := &libcni.RuntimeConf{
+		ContainerID: podSandboxID.ID,
+		NetNS:       podNetnsPath,
+		IfName:      network.DefaultInterfaceName,
+		Args: [][2]string{
+			{"IgnoreUnknown", "1"},
+			{"K8S_POD_NAMESPACE", podNs},
+			{"K8S_POD_NAME", podName},
+			{"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID},
+		},
+	}
+
+	// port mappings are a cni capability-based args, rather than parameters
+	// to a specific plugin
+	portMappings, err := plugin.host.GetPodPortMappings(podSandboxID.ID)
+	if err != nil {
+		return nil, fmt.Errorf("could not retrieve port mappings: %v", err)
+	}
+	portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
+	for _, p := range portMappings {
+		if p.HostPort <= 0 {
+			continue
+		}
+		portMappingsParam = append(portMappingsParam, cniPortMapping{
+			HostPort:      p.HostPort,
+			ContainerPort: p.ContainerPort,
+			Protocol:      strings.ToLower(string(p.Protocol)),
+			HostIP:        p.HostIP,
+		})
+	}
+	rt.CapabilityArgs = map[string]interface{}{
+		"portMappings": portMappingsParam,
+	}
+
+	return rt, nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/cni_others.go

@@ -0,0 +1,80 @@
+// +build !windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"fmt"
+
+	"github.com/containernetworking/cni/libcni"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+	loConfig, err := libcni.ConfListFromBytes([]byte(`{
+  "cniVersion": "0.2.0",
+  "name": "cni-loopback",
+  "plugins":[{
+    "type": "loopback"
+  }]
+}`))
+	if err != nil {
+		// The hardcoded config above should always be valid and unit tests will
+		// catch this
+		panic(err)
+	}
+	cninet := &libcni.CNIConfig{
+		Path: []string{vendorCNIDir(vendorDirPrefix, "loopback"), binDir},
+	}
+	loNetwork := &cniNetwork{
+		name:          "lo",
+		NetworkConfig: loConfig,
+		CNIConfig:     cninet,
+	}
+
+	return loNetwork
+}
+
+func (plugin *cniNetworkPlugin) platformInit() error {
+	var err error
+	plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
+// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
+func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+	}
+	if netnsPath == "" {
+		return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id)
+	}
+
+	ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &network.PodNetworkStatus{IP: ip}, nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/cni_test.go

@@ -0,0 +1,301 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"os"
+	"path"
+	"reflect"
+	"testing"
+	"text/template"
+
+	types020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/stretchr/testify/mock"
+	"k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	utiltesting "k8s.io/client-go/util/testing"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni/testing"
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
+	networktest "k8s.io/kubernetes/pkg/kubelet/network/testing"
+	"k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+func installPluginUnderTest(t *testing.T, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName string, plugName string) {
+	pluginDir := path.Join(testNetworkConfigPath, plugName)
+	err := os.MkdirAll(pluginDir, 0777)
+	if err != nil {
+		t.Fatalf("Failed to create plugin config dir: %v", err)
+	}
+	pluginConfig := path.Join(pluginDir, plugName+".conf")
+	f, err := os.Create(pluginConfig)
+	if err != nil {
+		t.Fatalf("Failed to install plugin")
+	}
+	networkConfig := fmt.Sprintf(`{ "name": "%s", "type": "%s", "capabilities": {"portMappings": true}  }`, plugName, vendorName)
+
+	_, err = f.WriteString(networkConfig)
+	if err != nil {
+		t.Fatalf("Failed to write network config file (%v)", err)
+	}
+	f.Close()
+
+	vendorCNIDir := fmt.Sprintf(VendorCNIDirTemplate, testVendorCNIDirPrefix, vendorName)
+	err = os.MkdirAll(vendorCNIDir, 0777)
+	if err != nil {
+		t.Fatalf("Failed to create plugin dir: %v", err)
+	}
+	pluginExec := path.Join(vendorCNIDir, vendorName)
+	f, err = os.Create(pluginExec)
+
+	const execScriptTempl = `#!/bin/bash
+cat > {{.InputFile}}
+env > {{.OutputEnv}}
+echo "%@" >> {{.OutputEnv}}
+export $(echo ${CNI_ARGS} | sed 's/;/ /g') &> /dev/null
+mkdir -p {{.OutputDir}} &> /dev/null
+echo -n "$CNI_COMMAND $CNI_NETNS $K8S_POD_NAMESPACE $K8S_POD_NAME $K8S_POD_INFRA_CONTAINER_ID" >& {{.OutputFile}}
+echo -n "{ \"ip4\": { \"ip\": \"10.1.0.23/24\" } }"
+`
+	execTemplateData := &map[string]interface{}{
+		"InputFile":  path.Join(pluginDir, plugName+".in"),
+		"OutputFile": path.Join(pluginDir, plugName+".out"),
+		"OutputEnv":  path.Join(pluginDir, plugName+".env"),
+		"OutputDir":  pluginDir,
+	}
+
+	tObj := template.Must(template.New("test").Parse(execScriptTempl))
+	buf := &bytes.Buffer{}
+	if err := tObj.Execute(buf, *execTemplateData); err != nil {
+		t.Fatalf("Error in executing script template - %v", err)
+	}
+	execScript := buf.String()
+	_, err = f.WriteString(execScript)
+	if err != nil {
+		t.Fatalf("Failed to write plugin exec - %v", err)
+	}
+
+	err = f.Chmod(0777)
+	if err != nil {
+		t.Fatalf("Failed to set exec perms on plugin")
+	}
+
+	f.Close()
+}
+
+func tearDownPlugin(tmpDir string) {
+	err := os.RemoveAll(tmpDir)
+	if err != nil {
+		fmt.Printf("Error in cleaning up test: %v", err)
+	}
+}
+
+type fakeNetworkHost struct {
+	networktest.FakePortMappingGetter
+	kubeClient clientset.Interface
+	runtime    kubecontainer.Runtime
+}
+
+func NewFakeHost(kubeClient clientset.Interface, pods []*containertest.FakePod, ports map[string][]*hostport.PortMapping) *fakeNetworkHost {
+	host := &fakeNetworkHost{
+		networktest.FakePortMappingGetter{PortMaps: ports},
+		kubeClient,
+		&containertest.FakeRuntime{
+			AllPodList: pods,
+		},
+	}
+	return host
+}
+
+func (fnh *fakeNetworkHost) GetPodByName(name, namespace string) (*v1.Pod, bool) {
+	return nil, false
+}
+
+func (fnh *fakeNetworkHost) GetKubeClient() clientset.Interface {
+	return fnh.kubeClient
+}
+
+func (fnh *fakeNetworkHost) GetRuntime() kubecontainer.Runtime {
+	return fnh.runtime
+}
+
+func (fnh *fakeNetworkHost) GetNetNS(containerID string) (string, error) {
+	return fnh.GetRuntime().GetNetNS(kubecontainer.ContainerID{Type: "test", ID: containerID})
+}
+
+func (fnh *fakeNetworkHost) SupportsLegacyFeatures() bool {
+	return true
+}
+
+func TestCNIPlugin(t *testing.T) {
+	// install some random plugin
+	pluginName := fmt.Sprintf("test%d", rand.Intn(1000))
+	vendorName := fmt.Sprintf("test_vendor%d", rand.Intn(1000))
+
+	podIP := "10.0.0.2"
+	podIPOutput := fmt.Sprintf("4: eth0    inet %s/24 scope global dynamic eth0\\       valid_lft forever preferred_lft forever", podIP)
+	fakeCmds := []fakeexec.FakeCommandAction{
+		func(cmd string, args ...string) exec.Cmd {
+			return fakeexec.InitFakeCmd(&fakeexec.FakeCmd{
+				CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+					func() ([]byte, error) {
+						return []byte(podIPOutput), nil
+					},
+				},
+			}, cmd, args...)
+		},
+	}
+
+	fexec := &fakeexec.FakeExec{
+		CommandScript: fakeCmds,
+		LookPathFunc: func(file string) (string, error) {
+			return fmt.Sprintf("/fake-bin/%s", file), nil
+		},
+	}
+
+	mockLoCNI := &mock_cni.MockCNI{}
+	// TODO mock for the test plugin too
+
+	tmpDir := utiltesting.MkTmpdirOrDie("cni-test")
+	testNetworkConfigPath := path.Join(tmpDir, "plugins", "net", "cni")
+	testVendorCNIDirPrefix := tmpDir
+	defer tearDownPlugin(tmpDir)
+	installPluginUnderTest(t, testVendorCNIDirPrefix, testNetworkConfigPath, vendorName, pluginName)
+
+	containerID := kubecontainer.ContainerID{Type: "test", ID: "test_infra_container"}
+	pods := []*containertest.FakePod{{
+		Pod: &kubecontainer.Pod{
+			Containers: []*kubecontainer.Container{
+				{ID: containerID},
+			},
+		},
+		NetnsPath: "/proc/12345/ns/net",
+	}}
+
+	plugins := probeNetworkPluginsWithVendorCNIDirPrefix(path.Join(testNetworkConfigPath, pluginName), "", testVendorCNIDirPrefix)
+	if len(plugins) != 1 {
+		t.Fatalf("Expected only one network plugin, got %d", len(plugins))
+	}
+	if plugins[0].Name() != "cni" {
+		t.Fatalf("Expected CNI network plugin, got %q", plugins[0].Name())
+	}
+
+	cniPlugin, ok := plugins[0].(*cniNetworkPlugin)
+	if !ok {
+		t.Fatalf("Not a CNI network plugin!")
+	}
+	cniPlugin.execer = fexec
+	cniPlugin.loNetwork.CNIConfig = mockLoCNI
+
+	mockLoCNI.On("AddNetworkList", cniPlugin.loNetwork.NetworkConfig, mock.AnythingOfType("*libcni.RuntimeConf")).Return(&types020.Result{IP4: &types020.IPConfig{IP: net.IPNet{IP: []byte{127, 0, 0, 1}}}}, nil)
+
+	ports := map[string][]*hostport.PortMapping{
+		containerID.ID: {
+			{
+				Name:          "name",
+				HostPort:      8008,
+				ContainerPort: 80,
+				Protocol:      "UDP",
+				HostIP:        "0.0.0.0",
+			},
+		},
+	}
+	fakeHost := NewFakeHost(nil, pods, ports)
+
+	plug, err := network.InitNetworkPlugin(plugins, "cni", fakeHost, kubeletconfig.HairpinNone, "10.0.0.0/8", network.UseDefaultMTU)
+	if err != nil {
+		t.Fatalf("Failed to select the desired plugin: %v", err)
+	}
+
+	// Set up the pod
+	err = plug.SetUpPod("podNamespace", "podName", containerID, map[string]string{})
+	if err != nil {
+		t.Errorf("Expected nil: %v", err)
+	}
+	outputEnv := path.Join(testNetworkConfigPath, pluginName, pluginName+".env")
+	eo, eerr := ioutil.ReadFile(outputEnv)
+	outputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".out")
+	output, err := ioutil.ReadFile(outputFile)
+	if err != nil || eerr != nil {
+		t.Errorf("Failed to read output file %s: %v (env %s err %v)", outputFile, err, eo, eerr)
+	}
+
+	expectedOutput := "ADD /proc/12345/ns/net podNamespace podName test_infra_container"
+	if string(output) != expectedOutput {
+		t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output))
+	}
+
+	// Verify the correct network configuration was passed
+	inputConfig := struct {
+		RuntimeConfig struct {
+			PortMappings []map[string]interface{} `json:"portMappings"`
+		} `json:"runtimeConfig"`
+	}{}
+	inputFile := path.Join(testNetworkConfigPath, pluginName, pluginName+".in")
+	inputBytes, inerr := ioutil.ReadFile(inputFile)
+	parseerr := json.Unmarshal(inputBytes, &inputConfig)
+	if inerr != nil || parseerr != nil {
+		t.Errorf("failed to parse reported cni input config %s: (%v %v)", inputFile, inerr, parseerr)
+	}
+	expectedMappings := []map[string]interface{}{
+		// hah, golang always unmarshals unstructured json numbers as float64
+		{"hostPort": 8008.0, "containerPort": 80.0, "protocol": "udp", "hostIP": "0.0.0.0"},
+	}
+	if !reflect.DeepEqual(inputConfig.RuntimeConfig.PortMappings, expectedMappings) {
+		t.Errorf("mismatch in expected port mappings. expected %v got %v", expectedMappings, inputConfig.RuntimeConfig.PortMappings)
+	}
+
+	// Get its IP address
+	status, err := plug.GetPodNetworkStatus("podNamespace", "podName", containerID)
+	if err != nil {
+		t.Errorf("Failed to read pod network status: %v", err)
+	}
+	if status.IP.String() != podIP {
+		t.Errorf("Expected pod IP %q but got %q", podIP, status.IP.String())
+	}
+
+	// Tear it down
+	err = plug.TearDownPod("podNamespace", "podName", containerID)
+	if err != nil {
+		t.Errorf("Expected nil: %v", err)
+	}
+	output, err = ioutil.ReadFile(path.Join(testNetworkConfigPath, pluginName, pluginName+".out"))
+	expectedOutput = "DEL /proc/12345/ns/net podNamespace podName test_infra_container"
+	if string(output) != expectedOutput {
+		t.Errorf("Mismatch in expected output for setup hook. Expected '%s', got '%s'", expectedOutput, string(output))
+	}
+
+	mockLoCNI.AssertExpectations(t)
+}
+
+func TestLoNetNonNil(t *testing.T) {
+	if conf := getLoNetwork("", ""); conf == nil {
+		t.Error("Expected non-nil lo network")
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/cni_windows.go

@@ -0,0 +1,61 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cni
+
+import (
+	"fmt"
+
+	cniTypes020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/golang/glog"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+func getLoNetwork(binDir, vendorDirPrefix string) *cniNetwork {
+	return nil
+}
+
+func (plugin *cniNetworkPlugin) platformInit() error {
+	return nil
+}
+
+// GetPodNetworkStatus : Assuming addToNetwork is idempotent, we can call this API as many times as required to get the IPAddress
+func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
+	}
+
+	result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
+
+	glog.V(5).Infof("GetPodNetworkStatus result %+v", result)
+	if err != nil {
+		glog.Errorf("error while adding to cni network: %s", err)
+		return nil, err
+	}
+
+	// Parse the result and get the IPAddress
+	var result020 *cniTypes020.Result
+	result020, err = cniTypes020.GetResult(result)
+	if err != nil {
+		glog.Errorf("error while cni parsing result: %s", err)
+		return nil, err
+	}
+	return &network.PodNetworkStatus{IP: result020.IP4.IP.IP}, nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/testing/BUILD

@@ -0,0 +1,30 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["mock_cni.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/cni/testing",
+    deps = [
+        "//vendor/github.com/containernetworking/cni/libcni:go_default_library",
+        "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
+        "//vendor/github.com/stretchr/testify/mock:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/cni/testing/mock_cni.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// mock_cni is a mock of the `libcni.CNI` interface. It's a handwritten mock
+// because there are only two functions to deal with.
+package mock_cni
+
+import (
+	"github.com/containernetworking/cni/libcni"
+	"github.com/containernetworking/cni/pkg/types"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockCNI struct {
+	mock.Mock
+}
+
+func (m *MockCNI) AddNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) (types.Result, error) {
+	args := m.Called(net, rt)
+	return args.Get(0).(types.Result), args.Error(1)
+}
+
+func (m *MockCNI) DelNetwork(net *libcni.NetworkConfig, rt *libcni.RuntimeConf) error {
+	args := m.Called(net, rt)
+	return args.Error(0)
+}
+
+func (m *MockCNI) DelNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) error {
+	args := m.Called(net, rt)
+	return args.Error(0)
+}
+
+func (m *MockCNI) AddNetworkList(net *libcni.NetworkConfigList, rt *libcni.RuntimeConf) (types.Result, error) {
+	args := m.Called(net, rt)
+	return args.Get(0).(types.Result), args.Error(1)
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/dns/BUILD

@@ -0,0 +1,51 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dns.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/dns",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/apis/core/validation:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/util/format:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["dns_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/dns",
+    library = ":go_default_library",
+    deps = [
+        "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/github.com/stretchr/testify/require:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/dns/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- sig-network-approvers
+reviewers:
+- sig-network-reviewers

vendor/k8s.io/kubernetes/pkg/kubelet/network/dns/dns.go

@@ -0,0 +1,414 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/api/core/v1"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/apis/core/validation"
+	"k8s.io/kubernetes/pkg/features"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/util/format"
+
+	"github.com/golang/glog"
+)
+
+var (
+	// The default dns opt strings.
+	defaultDNSOptions = []string{"ndots:5"}
+)
+
+type podDNSType int
+
+const (
+	podDNSCluster podDNSType = iota
+	podDNSHost
+	podDNSNone
+)
+
+// Configurer is used for setting up DNS resolver configuration when launching pods.
+type Configurer struct {
+	recorder record.EventRecorder
+	nodeRef  *v1.ObjectReference
+	nodeIP   net.IP
+
+	// If non-nil, use this for container DNS server.
+	clusterDNS []net.IP
+	// If non-empty, use this for container DNS search.
+	ClusterDomain string
+	// The path to the DNS resolver configuration file used as the base to generate
+	// the container's DNS resolver configuration file. This can be used in
+	// conjunction with clusterDomain and clusterDNS.
+	ResolverConfig string
+}
+
+// NewConfigurer returns a DNS configurer for launching pods.
+func NewConfigurer(recorder record.EventRecorder, nodeRef *v1.ObjectReference, nodeIP net.IP, clusterDNS []net.IP, clusterDomain, resolverConfig string) *Configurer {
+	return &Configurer{
+		recorder:       recorder,
+		nodeRef:        nodeRef,
+		nodeIP:         nodeIP,
+		clusterDNS:     clusterDNS,
+		ClusterDomain:  clusterDomain,
+		ResolverConfig: resolverConfig,
+	}
+}
+
+func omitDuplicates(strs []string) []string {
+	uniqueStrs := make(map[string]bool)
+
+	var ret []string
+	for _, str := range strs {
+		if !uniqueStrs[str] {
+			ret = append(ret, str)
+			uniqueStrs[str] = true
+		}
+	}
+	return ret
+}
+
+func (c *Configurer) formDNSSearchFitsLimits(composedSearch []string, pod *v1.Pod) []string {
+	limitsExceeded := false
+
+	if len(composedSearch) > validation.MaxDNSSearchPaths {
+		composedSearch = composedSearch[:validation.MaxDNSSearchPaths]
+		limitsExceeded = true
+	}
+
+	if resolvSearchLineStrLen := len(strings.Join(composedSearch, " ")); resolvSearchLineStrLen > validation.MaxDNSSearchListChars {
+		cutDomainsNum := 0
+		cutDomainsLen := 0
+		for i := len(composedSearch) - 1; i >= 0; i-- {
+			cutDomainsLen += len(composedSearch[i]) + 1
+			cutDomainsNum++
+
+			if (resolvSearchLineStrLen - cutDomainsLen) <= validation.MaxDNSSearchListChars {
+				break
+			}
+		}
+
+		composedSearch = composedSearch[:(len(composedSearch) - cutDomainsNum)]
+		limitsExceeded = true
+	}
+
+	if limitsExceeded {
+		log := fmt.Sprintf("Search Line limits were exceeded, some search paths have been omitted, the applied search line is: %s", strings.Join(composedSearch, " "))
+		c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", log)
+		glog.Error(log)
+	}
+	return composedSearch
+}
+
+func (c *Configurer) formDNSNameserversFitsLimits(nameservers []string, pod *v1.Pod) []string {
+	if len(nameservers) > validation.MaxDNSNameservers {
+		nameservers = nameservers[0:validation.MaxDNSNameservers]
+		log := fmt.Sprintf("Nameserver limits were exceeded, some nameservers have been omitted, the applied nameserver line is: %s", strings.Join(nameservers, " "))
+		c.recorder.Event(pod, v1.EventTypeWarning, "DNSConfigForming", log)
+		glog.Error(log)
+	}
+	return nameservers
+}
+
+func (c *Configurer) formDNSConfigFitsLimits(dnsConfig *runtimeapi.DNSConfig, pod *v1.Pod) *runtimeapi.DNSConfig {
+	dnsConfig.Servers = c.formDNSNameserversFitsLimits(dnsConfig.Servers, pod)
+	dnsConfig.Searches = c.formDNSSearchFitsLimits(dnsConfig.Searches, pod)
+	return dnsConfig
+}
+
+func (c *Configurer) generateSearchesForDNSClusterFirst(hostSearch []string, pod *v1.Pod) []string {
+	if c.ClusterDomain == "" {
+		return hostSearch
+	}
+
+	nsSvcDomain := fmt.Sprintf("%s.svc.%s", pod.Namespace, c.ClusterDomain)
+	svcDomain := fmt.Sprintf("svc.%s", c.ClusterDomain)
+	clusterSearch := []string{nsSvcDomain, svcDomain, c.ClusterDomain}
+
+	return omitDuplicates(append(clusterSearch, hostSearch...))
+}
+
+// CheckLimitsForResolvConf checks limits in resolv.conf.
+func (c *Configurer) CheckLimitsForResolvConf() {
+	f, err := os.Open(c.ResolverConfig)
+	if err != nil {
+		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error())
+		glog.Error("CheckLimitsForResolvConf: " + err.Error())
+		return
+	}
+	defer f.Close()
+
+	_, hostSearch, _, err := parseResolvConf(f)
+	if err != nil {
+		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", err.Error())
+		glog.Error("CheckLimitsForResolvConf: " + err.Error())
+		return
+	}
+
+	domainCountLimit := validation.MaxDNSSearchPaths
+
+	if c.ClusterDomain != "" {
+		domainCountLimit -= 3
+	}
+
+	if len(hostSearch) > domainCountLimit {
+		log := fmt.Sprintf("Resolv.conf file '%s' contains search line consisting of more than %d domains!", c.ResolverConfig, domainCountLimit)
+		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log)
+		glog.Error("CheckLimitsForResolvConf: " + log)
+		return
+	}
+
+	if len(strings.Join(hostSearch, " ")) > validation.MaxDNSSearchListChars {
+		log := fmt.Sprintf("Resolv.conf file '%s' contains search line which length is more than allowed %d chars!", c.ResolverConfig, validation.MaxDNSSearchListChars)
+		c.recorder.Event(c.nodeRef, v1.EventTypeWarning, "CheckLimitsForResolvConf", log)
+		glog.Error("CheckLimitsForResolvConf: " + log)
+		return
+	}
+
+	return
+}
+
+// parseResolveConf reads a resolv.conf file from the given reader, and parses
+// it into nameservers, searches and options, possibly returning an error.
+func parseResolvConf(reader io.Reader) (nameservers []string, searches []string, options []string, err error) {
+	file, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	// Lines of the form "nameserver 1.2.3.4" accumulate.
+	nameservers = []string{}
+
+	// Lines of the form "search example.com" overrule - last one wins.
+	searches = []string{}
+
+	// Lines of the form "option ndots:5 attempts:2" overrule - last one wins.
+	// Each option is recorded as an element in the array.
+	options = []string{}
+
+	lines := strings.Split(string(file), "\n")
+	for l := range lines {
+		trimmed := strings.TrimSpace(lines[l])
+		if strings.HasPrefix(trimmed, "#") {
+			continue
+		}
+		fields := strings.Fields(trimmed)
+		if len(fields) == 0 {
+			continue
+		}
+		if fields[0] == "nameserver" && len(fields) >= 2 {
+			nameservers = append(nameservers, fields[1])
+		}
+		if fields[0] == "search" {
+			searches = fields[1:]
+		}
+		if fields[0] == "options" {
+			options = fields[1:]
+		}
+	}
+
+	return nameservers, searches, options, nil
+}
+
+func (c *Configurer) getHostDNSConfig(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
+	var hostDNS, hostSearch, hostOptions []string
+	// Get host DNS settings
+	if c.ResolverConfig != "" {
+		f, err := os.Open(c.ResolverConfig)
+		if err != nil {
+			return nil, err
+		}
+		defer f.Close()
+
+		hostDNS, hostSearch, hostOptions, err = parseResolvConf(f)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &runtimeapi.DNSConfig{
+		Servers:  hostDNS,
+		Searches: hostSearch,
+		Options:  hostOptions,
+	}, nil
+}
+
+func getPodDNSType(pod *v1.Pod) (podDNSType, error) {
+	dnsPolicy := pod.Spec.DNSPolicy
+	switch dnsPolicy {
+	case v1.DNSNone:
+		if utilfeature.DefaultFeatureGate.Enabled(features.CustomPodDNS) {
+			return podDNSNone, nil
+		}
+		// This should not happen as kube-apiserver should have rejected
+		// setting dnsPolicy to DNSNone when feature gate is disabled.
+		return podDNSCluster, fmt.Errorf(fmt.Sprintf("invalid DNSPolicy=%v: custom pod DNS is disabled", dnsPolicy))
+	case v1.DNSClusterFirstWithHostNet:
+		return podDNSCluster, nil
+	case v1.DNSClusterFirst:
+		if !kubecontainer.IsHostNetworkPod(pod) {
+			return podDNSCluster, nil
+		}
+		// Fallback to DNSDefault for pod on hostnetowrk.
+		fallthrough
+	case v1.DNSDefault:
+		return podDNSHost, nil
+	}
+	// This should not happen as kube-apiserver should have rejected
+	// invalid dnsPolicy.
+	return podDNSCluster, fmt.Errorf(fmt.Sprintf("invalid DNSPolicy=%v", dnsPolicy))
+}
+
+// Merge DNS options. If duplicated, entries given by PodDNSConfigOption will
+// overwrite the existing ones.
+func mergeDNSOptions(existingDNSConfigOptions []string, dnsConfigOptions []v1.PodDNSConfigOption) []string {
+	optionsMap := make(map[string]string)
+	for _, op := range existingDNSConfigOptions {
+		if index := strings.Index(op, ":"); index != -1 {
+			optionsMap[op[:index]] = op[index+1:]
+		} else {
+			optionsMap[op] = ""
+		}
+	}
+	for _, op := range dnsConfigOptions {
+		if op.Value != nil {
+			optionsMap[op.Name] = *op.Value
+		} else {
+			optionsMap[op.Name] = ""
+		}
+	}
+	// Reconvert DNS options into a string array.
+	options := []string{}
+	for opName, opValue := range optionsMap {
+		op := opName
+		if opValue != "" {
+			op = op + ":" + opValue
+		}
+		options = append(options, op)
+	}
+	return options
+}
+
+// appendDNSConfig appends DNS servers, search paths and options given by
+// PodDNSConfig to the existing DNS config. Duplicated entries will be merged.
+// This assumes existingDNSConfig and dnsConfig are not nil.
+func appendDNSConfig(existingDNSConfig *runtimeapi.DNSConfig, dnsConfig *v1.PodDNSConfig) *runtimeapi.DNSConfig {
+	existingDNSConfig.Servers = omitDuplicates(append(existingDNSConfig.Servers, dnsConfig.Nameservers...))
+	existingDNSConfig.Searches = omitDuplicates(append(existingDNSConfig.Searches, dnsConfig.Searches...))
+	existingDNSConfig.Options = mergeDNSOptions(existingDNSConfig.Options, dnsConfig.Options)
+	return existingDNSConfig
+}
+
+// GetPodDNS returns DNS setttings for the pod.
+func (c *Configurer) GetPodDNS(pod *v1.Pod) (*runtimeapi.DNSConfig, error) {
+	dnsConfig, err := c.getHostDNSConfig(pod)
+	if err != nil {
+		return nil, err
+	}
+
+	dnsType, err := getPodDNSType(pod)
+	if err != nil {
+		glog.Errorf("Failed to get DNS type for pod %q: %v. Falling back to DNSClusterFirst policy.", format.Pod(pod), err)
+		dnsType = podDNSCluster
+	}
+	switch dnsType {
+	case podDNSNone:
+		// DNSNone should use empty DNS settings as the base.
+		dnsConfig = &runtimeapi.DNSConfig{}
+	case podDNSCluster:
+		if len(c.clusterDNS) != 0 {
+			// For a pod with DNSClusterFirst policy, the cluster DNS server is
+			// the only nameserver configured for the pod. The cluster DNS server
+			// itself will forward queries to other nameservers that is configured
+			// to use, in case the cluster DNS server cannot resolve the DNS query
+			// itself.
+			dnsConfig.Servers = []string{}
+			for _, ip := range c.clusterDNS {
+				dnsConfig.Servers = append(dnsConfig.Servers, ip.String())
+			}
+			dnsConfig.Searches = c.generateSearchesForDNSClusterFirst(dnsConfig.Searches, pod)
+			dnsConfig.Options = defaultDNSOptions
+			break
+		}
+		// clusterDNS is not known. Pod with ClusterDNSFirst Policy cannot be created.
+		nodeErrorMsg := fmt.Sprintf("kubelet does not have ClusterDNS IP configured and cannot create Pod using %q policy. Falling back to %q policy.", v1.DNSClusterFirst, v1.DNSDefault)
+		c.recorder.Eventf(c.nodeRef, v1.EventTypeWarning, "MissingClusterDNS", nodeErrorMsg)
+		c.recorder.Eventf(pod, v1.EventTypeWarning, "MissingClusterDNS", "pod: %q. %s", format.Pod(pod), nodeErrorMsg)
+		// Fallback to DNSDefault.
+		fallthrough
+	case podDNSHost:
+		// When the kubelet --resolv-conf flag is set to the empty string, use
+		// DNS settings that override the docker default (which is to use
+		// /etc/resolv.conf) and effectively disable DNS lookups. According to
+		// the bind documentation, the behavior of the DNS client library when
+		// "nameservers" are not specified is to "use the nameserver on the
+		// local machine". A nameserver setting of localhost is equivalent to
+		// this documented behavior.
+		if c.ResolverConfig == "" {
+			switch {
+			case c.nodeIP == nil || c.nodeIP.To4() != nil:
+				dnsConfig.Servers = []string{"127.0.0.1"}
+			case c.nodeIP.To16() != nil:
+				dnsConfig.Servers = []string{"::1"}
+			}
+			dnsConfig.Searches = []string{"."}
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.CustomPodDNS) && pod.Spec.DNSConfig != nil {
+		dnsConfig = appendDNSConfig(dnsConfig, pod.Spec.DNSConfig)
+	}
+	return c.formDNSConfigFitsLimits(dnsConfig, pod), nil
+}
+
+// SetupDNSinContainerizedMounter replaces the nameserver in containerized-mounter's rootfs/etc/resolve.conf with kubelet.ClusterDNS
+func (c *Configurer) SetupDNSinContainerizedMounter(mounterPath string) {
+	resolvePath := filepath.Join(strings.TrimSuffix(mounterPath, "/mounter"), "rootfs", "etc", "resolv.conf")
+	dnsString := ""
+	for _, dns := range c.clusterDNS {
+		dnsString = dnsString + fmt.Sprintf("nameserver %s\n", dns)
+	}
+	if c.ResolverConfig != "" {
+		f, err := os.Open(c.ResolverConfig)
+		defer f.Close()
+		if err != nil {
+			glog.Error("Could not open resolverConf file")
+		} else {
+			_, hostSearch, _, err := parseResolvConf(f)
+			if err != nil {
+				glog.Errorf("Error for parsing the reslov.conf file: %v", err)
+			} else {
+				dnsString = dnsString + "search"
+				for _, search := range hostSearch {
+					dnsString = dnsString + fmt.Sprintf(" %s", search)
+				}
+				dnsString = dnsString + "\n"
+			}
+		}
+	}
+	if err := ioutil.WriteFile(resolvePath, []byte(dnsString), 0600); err != nil {
+		glog.Errorf("Could not write dns nameserver in file %s, with error %v", resolvePath, err)
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/dns/dns_test.go

@@ -0,0 +1,601 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dns
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/tools/record"
+	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	fetchEvent = func(recorder *record.FakeRecorder) string {
+		select {
+		case event := <-recorder.Events:
+			return event
+		default:
+			return ""
+		}
+	}
+)
+
+func TestParseResolvConf(t *testing.T) {
+	testCases := []struct {
+		data        string
+		nameservers []string
+		searches    []string
+		options     []string
+	}{
+		{"", []string{}, []string{}, []string{}},
+		{" ", []string{}, []string{}, []string{}},
+		{"\n", []string{}, []string{}, []string{}},
+		{"\t\n\t", []string{}, []string{}, []string{}},
+		{"#comment\n", []string{}, []string{}, []string{}},
+		{" #comment\n", []string{}, []string{}, []string{}},
+		{"#comment\n#comment", []string{}, []string{}, []string{}},
+		{"#comment\nnameserver", []string{}, []string{}, []string{}},
+		{"#comment\nnameserver\nsearch", []string{}, []string{}, []string{}},
+		{"nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
+		{" nameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
+		{"\tnameserver 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
+		{"nameserver\t1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
+		{"nameserver \t 1.2.3.4", []string{"1.2.3.4"}, []string{}, []string{}},
+		{"nameserver 1.2.3.4\nnameserver 5.6.7.8", []string{"1.2.3.4", "5.6.7.8"}, []string{}, []string{}},
+		{"nameserver 1.2.3.4 #comment", []string{"1.2.3.4"}, []string{}, []string{}},
+		{"search foo", []string{}, []string{"foo"}, []string{}},
+		{"search foo bar", []string{}, []string{"foo", "bar"}, []string{}},
+		{"search foo bar bat\n", []string{}, []string{"foo", "bar", "bat"}, []string{}},
+		{"search foo\nsearch bar", []string{}, []string{"bar"}, []string{}},
+		{"nameserver 1.2.3.4\nsearch foo bar", []string{"1.2.3.4"}, []string{"foo", "bar"}, []string{}},
+		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}, []string{}},
+		{"#comment\nnameserver 1.2.3.4\n#comment\nsearch foo\ncomment", []string{"1.2.3.4"}, []string{"foo"}, []string{}},
+		{"options ndots:5 attempts:2", []string{}, []string{}, []string{"ndots:5", "attempts:2"}},
+		{"options ndots:1\noptions ndots:5 attempts:3", []string{}, []string{}, []string{"ndots:5", "attempts:3"}},
+		{"nameserver 1.2.3.4\nsearch foo\nnameserver 5.6.7.8\nsearch bar\noptions ndots:5 attempts:4", []string{"1.2.3.4", "5.6.7.8"}, []string{"bar"}, []string{"ndots:5", "attempts:4"}},
+	}
+	for i, tc := range testCases {
+		ns, srch, opts, err := parseResolvConf(strings.NewReader(tc.data))
+		require.NoError(t, err)
+		assert.EqualValues(t, tc.nameservers, ns, "test case [%d]: name servers", i)
+		assert.EqualValues(t, tc.searches, srch, "test case [%d] searches", i)
+		assert.EqualValues(t, tc.options, opts, "test case [%d] options", i)
+	}
+}
+
+func TestFormDNSSearchFitsLimits(t *testing.T) {
+	recorder := record.NewFakeRecorder(20)
+	nodeRef := &v1.ObjectReference{
+		Kind:      "Node",
+		Name:      string("testNode"),
+		UID:       types.UID("testNode"),
+		Namespace: "",
+	}
+	testClusterDNSDomain := "TEST"
+
+	configurer := NewConfigurer(recorder, nodeRef, nil, nil, testClusterDNSDomain, "")
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:         "",
+			Name:        "test_pod",
+			Namespace:   "testNS",
+			Annotations: map[string]string{},
+		},
+	}
+
+	testCases := []struct {
+		hostNames    []string
+		resultSearch []string
+		events       []string
+	}{
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST"},
+			[]string{},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB"},
+			[]string{},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", strings.Repeat("B", 256), "BBB"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA"},
+			[]string{"Search Line limits were exceeded, some search paths have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA"},
+		},
+
+		{
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB", "CCC", "DDD"},
+			[]string{"testNS.svc.TEST", "svc.TEST", "TEST", "AAA", "BBB", "CCC"},
+			[]string{"Search Line limits were exceeded, some search paths have been omitted, the applied search line is: testNS.svc.TEST svc.TEST TEST AAA BBB CCC"},
+		},
+	}
+
+	for i, tc := range testCases {
+		dnsSearch := configurer.formDNSSearchFitsLimits(tc.hostNames, pod)
+		assert.EqualValues(t, tc.resultSearch, dnsSearch, "test [%d]", i)
+		for _, expectedEvent := range tc.events {
+			expected := fmt.Sprintf("%s %s %s", v1.EventTypeWarning, "DNSConfigForming", expectedEvent)
+			event := fetchEvent(recorder)
+			assert.Equal(t, expected, event, "test [%d]", i)
+		}
+	}
+}
+
+func TestFormDNSNameserversFitsLimits(t *testing.T) {
+	recorder := record.NewFakeRecorder(20)
+	nodeRef := &v1.ObjectReference{
+		Kind:      "Node",
+		Name:      string("testNode"),
+		UID:       types.UID("testNode"),
+		Namespace: "",
+	}
+	testClusterDNSDomain := "TEST"
+
+	configurer := NewConfigurer(recorder, nodeRef, nil, nil, testClusterDNSDomain, "")
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:         "",
+			Name:        "test_pod",
+			Namespace:   "testNS",
+			Annotations: map[string]string{},
+		},
+	}
+
+	testCases := []struct {
+		desc               string
+		nameservers        []string
+		expectedNameserver []string
+		expectedEvent      bool
+	}{
+		{
+			desc:               "valid: 1 nameserver",
+			nameservers:        []string{"127.0.0.1"},
+			expectedNameserver: []string{"127.0.0.1"},
+			expectedEvent:      false,
+		},
+		{
+			desc:               "valid: 3 nameservers",
+			nameservers:        []string{"127.0.0.1", "10.0.0.10", "8.8.8.8"},
+			expectedNameserver: []string{"127.0.0.1", "10.0.0.10", "8.8.8.8"},
+			expectedEvent:      false,
+		},
+		{
+			desc:               "invalid: 4 nameservers, trimmed to 3",
+			nameservers:        []string{"127.0.0.1", "10.0.0.10", "8.8.8.8", "1.2.3.4"},
+			expectedNameserver: []string{"127.0.0.1", "10.0.0.10", "8.8.8.8"},
+			expectedEvent:      true,
+		},
+	}
+
+	for _, tc := range testCases {
+		appliedNameservers := configurer.formDNSNameserversFitsLimits(tc.nameservers, pod)
+		assert.EqualValues(t, tc.expectedNameserver, appliedNameservers, tc.desc)
+		event := fetchEvent(recorder)
+		if tc.expectedEvent && len(event) == 0 {
+			t.Errorf("%s: formDNSNameserversFitsLimits(%v) expected event, got no event.", tc.desc, tc.nameservers)
+		} else if !tc.expectedEvent && len(event) > 0 {
+			t.Errorf("%s: formDNSNameserversFitsLimits(%v) expected no event, got event: %v", tc.desc, tc.nameservers, event)
+		}
+	}
+}
+
+func TestMergeDNSOptions(t *testing.T) {
+	testOptionValue := "3"
+
+	testCases := []struct {
+		desc                     string
+		existingDNSConfigOptions []string
+		dnsConfigOptions         []v1.PodDNSConfigOption
+		expectedOptions          []string
+	}{
+		{
+			desc: "Empty dnsConfigOptions",
+			existingDNSConfigOptions: []string{"ndots:5", "debug"},
+			dnsConfigOptions:         nil,
+			expectedOptions:          []string{"ndots:5", "debug"},
+		},
+		{
+			desc: "No duplicated entries",
+			existingDNSConfigOptions: []string{"ndots:5", "debug"},
+			dnsConfigOptions: []v1.PodDNSConfigOption{
+				{Name: "single-request"},
+				{Name: "attempts", Value: &testOptionValue},
+			},
+			expectedOptions: []string{"ndots:5", "debug", "single-request", "attempts:3"},
+		},
+		{
+			desc: "Overwrite duplicated entries",
+			existingDNSConfigOptions: []string{"ndots:5", "debug"},
+			dnsConfigOptions: []v1.PodDNSConfigOption{
+				{Name: "ndots", Value: &testOptionValue},
+				{Name: "debug"},
+				{Name: "single-request"},
+				{Name: "attempts", Value: &testOptionValue},
+			},
+			expectedOptions: []string{"ndots:3", "debug", "single-request", "attempts:3"},
+		},
+	}
+
+	for _, tc := range testCases {
+		options := mergeDNSOptions(tc.existingDNSConfigOptions, tc.dnsConfigOptions)
+		// Options order may be changed after conversion.
+		if !sets.NewString(options...).Equal(sets.NewString(tc.expectedOptions...)) {
+			t.Errorf("%s: mergeDNSOptions(%v, %v)=%v, want %v", tc.desc, tc.existingDNSConfigOptions, tc.dnsConfigOptions, options, tc.expectedOptions)
+		}
+	}
+}
+
+func TestGetPodDNSType(t *testing.T) {
+	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
+	defer func() {
+		// Restoring the old value.
+		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
+			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+		}
+	}()
+
+	recorder := record.NewFakeRecorder(20)
+	nodeRef := &v1.ObjectReference{
+		Kind:      "Node",
+		Name:      string("testNode"),
+		UID:       types.UID("testNode"),
+		Namespace: "",
+	}
+	testClusterDNSDomain := "TEST"
+	clusterNS := "203.0.113.1"
+	testClusterDNS := []net.IP{net.ParseIP(clusterNS)}
+
+	configurer := NewConfigurer(recorder, nodeRef, nil, nil, testClusterDNSDomain, "")
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:         "",
+			Name:        "test_pod",
+			Namespace:   "testNS",
+			Annotations: map[string]string{},
+		},
+	}
+
+	testCases := []struct {
+		desc                    string
+		customPodDNSFeatureGate bool
+		hasClusterDNS           bool
+		hostNetwork             bool
+		dnsPolicy               v1.DNSPolicy
+		expectedDNSType         podDNSType
+		expectedError           bool
+	}{
+		{
+			desc:            "valid DNSClusterFirst without hostnetwork",
+			hasClusterDNS:   true,
+			dnsPolicy:       v1.DNSClusterFirst,
+			expectedDNSType: podDNSCluster,
+		},
+		{
+			desc:            "valid DNSClusterFirstWithHostNet with hostnetwork",
+			hasClusterDNS:   true,
+			hostNetwork:     true,
+			dnsPolicy:       v1.DNSClusterFirstWithHostNet,
+			expectedDNSType: podDNSCluster,
+		},
+		{
+			desc:            "valid DNSClusterFirstWithHostNet without hostnetwork",
+			hasClusterDNS:   true,
+			dnsPolicy:       v1.DNSClusterFirstWithHostNet,
+			expectedDNSType: podDNSCluster,
+		},
+		{
+			desc:            "valid DNSDefault without hostnetwork",
+			dnsPolicy:       v1.DNSDefault,
+			expectedDNSType: podDNSHost,
+		},
+		{
+			desc:            "valid DNSDefault with hostnetwork",
+			hostNetwork:     true,
+			dnsPolicy:       v1.DNSDefault,
+			expectedDNSType: podDNSHost,
+		},
+		{
+			desc:            "DNSClusterFirst with hostnetwork, fallback to DNSDefault",
+			hasClusterDNS:   true,
+			hostNetwork:     true,
+			dnsPolicy:       v1.DNSClusterFirst,
+			expectedDNSType: podDNSHost,
+		},
+		{
+			desc: "valid DNSNone with feature gate",
+			customPodDNSFeatureGate: true,
+			dnsPolicy:               v1.DNSNone,
+			expectedDNSType:         podDNSNone,
+		},
+		{
+			desc:          "DNSNone without feature gate, should return error",
+			dnsPolicy:     v1.DNSNone,
+			expectedError: true,
+		},
+		{
+			desc:          "invalid DNS policy, should return error",
+			dnsPolicy:     "invalidPolicy",
+			expectedError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", tc.customPodDNSFeatureGate)); err != nil {
+			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+		}
+
+		if tc.hasClusterDNS {
+			configurer.clusterDNS = testClusterDNS
+		} else {
+			configurer.clusterDNS = nil
+		}
+		pod.Spec.DNSPolicy = tc.dnsPolicy
+		pod.Spec.HostNetwork = tc.hostNetwork
+
+		resType, err := getPodDNSType(pod)
+		if tc.expectedError {
+			if err == nil {
+				t.Errorf("%s: GetPodDNSType(%v) got no error, want error", tc.desc, pod)
+			}
+			continue
+		}
+		if resType != tc.expectedDNSType {
+			t.Errorf("%s: GetPodDNSType(%v)=%v, want %v", tc.desc, pod, resType, tc.expectedDNSType)
+		}
+	}
+}
+
+func TestGetPodDNS(t *testing.T) {
+	recorder := record.NewFakeRecorder(20)
+	nodeRef := &v1.ObjectReference{
+		Kind:      "Node",
+		Name:      string("testNode"),
+		UID:       types.UID("testNode"),
+		Namespace: "",
+	}
+	clusterNS := "203.0.113.1"
+	testClusterDNSDomain := "kubernetes.io"
+	testClusterDNS := []net.IP{net.ParseIP(clusterNS)}
+
+	configurer := NewConfigurer(recorder, nodeRef, nil, testClusterDNS, testClusterDNSDomain, "")
+
+	pods := newTestPods(4)
+	pods[0].Spec.DNSPolicy = v1.DNSClusterFirstWithHostNet
+	pods[1].Spec.DNSPolicy = v1.DNSClusterFirst
+	pods[2].Spec.DNSPolicy = v1.DNSClusterFirst
+	pods[2].Spec.HostNetwork = false
+	pods[3].Spec.DNSPolicy = v1.DNSDefault
+
+	options := make([]struct {
+		DNS       []string
+		DNSSearch []string
+	}, 4)
+	for i, pod := range pods {
+		var err error
+		dnsConfig, err := configurer.GetPodDNS(pod)
+		if err != nil {
+			t.Fatalf("failed to generate container options: %v", err)
+		}
+		options[i].DNS, options[i].DNSSearch = dnsConfig.Servers, dnsConfig.Searches
+	}
+	if len(options[0].DNS) != 1 || options[0].DNS[0] != clusterNS {
+		t.Errorf("expected nameserver %s, got %+v", clusterNS, options[0].DNS)
+	}
+	if len(options[0].DNSSearch) == 0 || options[0].DNSSearch[0] != ".svc."+configurer.ClusterDomain {
+		t.Errorf("expected search %s, got %+v", ".svc."+configurer.ClusterDomain, options[0].DNSSearch)
+	}
+	if len(options[1].DNS) != 1 || options[1].DNS[0] != "127.0.0.1" {
+		t.Errorf("expected nameserver 127.0.0.1, got %+v", options[1].DNS)
+	}
+	if len(options[1].DNSSearch) != 1 || options[1].DNSSearch[0] != "." {
+		t.Errorf("expected search \".\", got %+v", options[1].DNSSearch)
+	}
+	if len(options[2].DNS) != 1 || options[2].DNS[0] != clusterNS {
+		t.Errorf("expected nameserver %s, got %+v", clusterNS, options[2].DNS)
+	}
+	if len(options[2].DNSSearch) == 0 || options[2].DNSSearch[0] != ".svc."+configurer.ClusterDomain {
+		t.Errorf("expected search %s, got %+v", ".svc."+configurer.ClusterDomain, options[2].DNSSearch)
+	}
+	if len(options[3].DNS) != 1 || options[3].DNS[0] != "127.0.0.1" {
+		t.Errorf("expected nameserver 127.0.0.1, got %+v", options[3].DNS)
+	}
+	if len(options[3].DNSSearch) != 1 || options[3].DNSSearch[0] != "." {
+		t.Errorf("expected search \".\", got %+v", options[3].DNSSearch)
+	}
+
+	testResolverConfig := "/etc/resolv.conf"
+	configurer = NewConfigurer(recorder, nodeRef, nil, testClusterDNS, testClusterDNSDomain, testResolverConfig)
+	for i, pod := range pods {
+		var err error
+		dnsConfig, err := configurer.GetPodDNS(pod)
+		if err != nil {
+			t.Fatalf("failed to generate container options: %v", err)
+		}
+		options[i].DNS, options[i].DNSSearch = dnsConfig.Servers, dnsConfig.Searches
+	}
+	t.Logf("nameservers %+v", options[1].DNS)
+	if len(options[0].DNS) != 1 {
+		t.Errorf("expected cluster nameserver only, got %+v", options[0].DNS)
+	} else if options[0].DNS[0] != clusterNS {
+		t.Errorf("expected nameserver %s, got %v", clusterNS, options[0].DNS[0])
+	}
+	expLength := len(options[1].DNSSearch) + 3
+	if expLength > 6 {
+		expLength = 6
+	}
+	if len(options[0].DNSSearch) != expLength {
+		t.Errorf("expected prepend of cluster domain, got %+v", options[0].DNSSearch)
+	} else if options[0].DNSSearch[0] != ".svc."+configurer.ClusterDomain {
+		t.Errorf("expected domain %s, got %s", ".svc."+configurer.ClusterDomain, options[0].DNSSearch)
+	}
+	if len(options[2].DNS) != 1 {
+		t.Errorf("expected cluster nameserver only, got %+v", options[2].DNS)
+	} else if options[2].DNS[0] != clusterNS {
+		t.Errorf("expected nameserver %s, got %v", clusterNS, options[2].DNS[0])
+	}
+	if len(options[2].DNSSearch) != expLength {
+		t.Errorf("expected prepend of cluster domain, got %+v", options[2].DNSSearch)
+	} else if options[2].DNSSearch[0] != ".svc."+configurer.ClusterDomain {
+		t.Errorf("expected domain %s, got %s", ".svc."+configurer.ClusterDomain, options[0].DNSSearch)
+	}
+}
+
+func TestGetPodDNSCustom(t *testing.T) {
+	customDNSEnabled := utilfeature.DefaultFeatureGate.Enabled("CustomPodDNS")
+	defer func() {
+		// Restoring the old value.
+		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", customDNSEnabled)); err != nil {
+			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+		}
+	}()
+
+	recorder := record.NewFakeRecorder(20)
+	nodeRef := &v1.ObjectReference{
+		Kind:      "Node",
+		Name:      string("testNode"),
+		UID:       types.UID("testNode"),
+		Namespace: "",
+	}
+	clusterNS := "203.0.113.1"
+	testClusterDNSDomain := "kubernetes.io"
+	testClusterDNS := []net.IP{net.ParseIP(clusterNS)}
+	testOptionValue := "3"
+
+	configurer := NewConfigurer(recorder, nodeRef, nil, testClusterDNS, testClusterDNSDomain, "")
+
+	pod := &v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			UID:         "",
+			Name:        "test_pod",
+			Namespace:   "testNS",
+			Annotations: map[string]string{},
+		},
+		Spec: v1.PodSpec{
+			DNSPolicy: v1.DNSClusterFirst,
+		},
+	}
+	clusterFirstDNSConfig, err := configurer.GetPodDNS(pod)
+	if err != nil {
+		t.Fatalf("Preparing clusterFirstDNSConfig: GetPodDNS(%v), unexpected error: %v", pod, err)
+	}
+
+	// Overwrite DNSPolicy for testing.
+	pod.Spec.DNSPolicy = v1.DNSNone
+
+	testCases := []struct {
+		desc                    string
+		customPodDNSFeatureGate bool
+		dnsConfig               *v1.PodDNSConfig
+		expectedDNSConfig       *runtimeapi.DNSConfig
+	}{
+		{
+			desc:              "feature gate is disabled, DNSNone should fallback to DNSClusterFirst",
+			expectedDNSConfig: clusterFirstDNSConfig,
+		},
+		{
+			desc: "feature gate is enabled, DNSNone without DNSConfig should have empty DNS settings",
+			customPodDNSFeatureGate: true,
+			expectedDNSConfig:       &runtimeapi.DNSConfig{},
+		},
+		{
+			desc: "feature gate is enabled, DNSNone with DNSConfig should have a merged DNS settings",
+			customPodDNSFeatureGate: true,
+			dnsConfig: &v1.PodDNSConfig{
+				Nameservers: []string{"10.0.0.10"},
+				Searches:    []string{"my.domain", "second.domain"},
+				Options: []v1.PodDNSConfigOption{
+					{Name: "ndots", Value: &testOptionValue},
+					{Name: "debug"},
+				},
+			},
+			expectedDNSConfig: &runtimeapi.DNSConfig{
+				Servers:  []string{"10.0.0.10"},
+				Searches: []string{"my.domain", "second.domain"},
+				Options:  []string{"ndots:3", "debug"},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		if err := utilfeature.DefaultFeatureGate.Set(fmt.Sprintf("CustomPodDNS=%v", tc.customPodDNSFeatureGate)); err != nil {
+			t.Errorf("Failed to set CustomPodDNS feature gate: %v", err)
+		}
+
+		pod.Spec.DNSConfig = tc.dnsConfig
+
+		resDNSConfig, err := configurer.GetPodDNS(pod)
+		if err != nil {
+			t.Errorf("%s: GetPodDNS(%v), unexpected error: %v", tc.desc, pod, err)
+		}
+		if !dnsConfigsAreEqual(resDNSConfig, tc.expectedDNSConfig) {
+			t.Errorf("%s: GetPodDNS(%v)=%v, want %v", tc.desc, pod, resDNSConfig, tc.expectedDNSConfig)
+		}
+	}
+}
+
+func dnsConfigsAreEqual(resConfig, expectedConfig *runtimeapi.DNSConfig) bool {
+	if len(resConfig.Servers) != len(expectedConfig.Servers) ||
+		len(resConfig.Searches) != len(expectedConfig.Searches) ||
+		len(resConfig.Options) != len(expectedConfig.Options) {
+		return false
+	}
+	for i, server := range resConfig.Servers {
+		if expectedConfig.Servers[i] != server {
+			return false
+		}
+	}
+	for i, search := range resConfig.Searches {
+		if expectedConfig.Searches[i] != search {
+			return false
+		}
+	}
+	// Options order may be changed after conversion.
+	return sets.NewString(resConfig.Options...).Equal(sets.NewString(expectedConfig.Options...))
+}
+
+func newTestPods(count int) []*v1.Pod {
+	pods := make([]*v1.Pod, count)
+	for i := 0; i < count; i++ {
+		pods[i] = &v1.Pod{
+			Spec: v1.PodSpec{
+				HostNetwork: true,
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				UID:  types.UID(10000 + i),
+				Name: fmt.Sprintf("pod%d", i),
+			},
+		}
+	}
+	return pods
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hairpin/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["hairpin.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/hairpin",
+    deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/utils/exec:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["hairpin_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/hairpin",
+    library = ":go_default_library",
+    deps = [
+        "//vendor/k8s.io/utils/exec:go_default_library",
+        "//vendor/k8s.io/utils/exec/testing:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/hairpin/hairpin.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hairpin
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path"
+	"regexp"
+	"strconv"
+
+	"github.com/golang/glog"
+	"k8s.io/utils/exec"
+)
+
+const (
+	sysfsNetPath            = "/sys/devices/virtual/net"
+	brportRelativePath      = "brport"
+	hairpinModeRelativePath = "hairpin_mode"
+	hairpinEnable           = "1"
+)
+
+var (
+	ethtoolOutputRegex = regexp.MustCompile("peer_ifindex: (\\d+)")
+)
+
+func SetUpContainerPid(containerPid int, containerInterfaceName string) error {
+	pidStr := fmt.Sprintf("%d", containerPid)
+	nsenterArgs := []string{"-t", pidStr, "-n"}
+	return setUpContainerInternal(containerInterfaceName, pidStr, nsenterArgs)
+}
+
+func SetUpContainerPath(netnsPath string, containerInterfaceName string) error {
+	if netnsPath[0] != '/' {
+		return fmt.Errorf("netnsPath path '%s' was invalid", netnsPath)
+	}
+	nsenterArgs := []string{"--net=" + netnsPath}
+	return setUpContainerInternal(containerInterfaceName, netnsPath, nsenterArgs)
+}
+
+func setUpContainerInternal(containerInterfaceName, containerDesc string, nsenterArgs []string) error {
+	e := exec.New()
+	hostIfName, err := findPairInterfaceOfContainerInterface(e, containerInterfaceName, containerDesc, nsenterArgs)
+	if err != nil {
+		return err
+	}
+	return setUpInterface(hostIfName)
+}
+
+func findPairInterfaceOfContainerInterface(e exec.Interface, containerInterfaceName, containerDesc string, nsenterArgs []string) (string, error) {
+	nsenterPath, err := e.LookPath("nsenter")
+	if err != nil {
+		return "", err
+	}
+	ethtoolPath, err := e.LookPath("ethtool")
+	if err != nil {
+		return "", err
+	}
+
+	nsenterArgs = append(nsenterArgs, "-F", "--", ethtoolPath, "--statistics", containerInterfaceName)
+	output, err := e.Command(nsenterPath, nsenterArgs...).CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("Unable to query interface %s of container %s: %v: %s", containerInterfaceName, containerDesc, err, string(output))
+	}
+	// look for peer_ifindex
+	match := ethtoolOutputRegex.FindSubmatch(output)
+	if match == nil {
+		return "", fmt.Errorf("No peer_ifindex in interface statistics for %s of container %s", containerInterfaceName, containerDesc)
+	}
+	peerIfIndex, err := strconv.Atoi(string(match[1]))
+	if err != nil { // seems impossible (\d+ not numeric)
+		return "", fmt.Errorf("peer_ifindex wasn't numeric: %s: %v", match[1], err)
+	}
+	iface, err := net.InterfaceByIndex(peerIfIndex)
+	if err != nil {
+		return "", err
+	}
+	return iface.Name, nil
+}
+
+func setUpInterface(ifName string) error {
+	glog.V(3).Infof("Enabling hairpin on interface %s", ifName)
+	ifPath := path.Join(sysfsNetPath, ifName)
+	if _, err := os.Stat(ifPath); err != nil {
+		return err
+	}
+	brportPath := path.Join(ifPath, brportRelativePath)
+	if _, err := os.Stat(brportPath); err != nil && os.IsNotExist(err) {
+		// Device is not on a bridge, so doesn't need hairpin mode
+		return nil
+	}
+	hairpinModeFile := path.Join(brportPath, hairpinModeRelativePath)
+	return ioutil.WriteFile(hairpinModeFile, []byte(hairpinEnable), 0644)
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hairpin/hairpin_test.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hairpin
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"testing"
+
+	"k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+func TestFindPairInterfaceOfContainerInterface(t *testing.T) {
+	// there should be at least "lo" on any system
+	interfaces, _ := net.Interfaces()
+	validOutput := fmt.Sprintf("garbage\n   peer_ifindex: %d", interfaces[0].Index)
+	invalidOutput := fmt.Sprintf("garbage\n   unknown: %d", interfaces[0].Index)
+
+	tests := []struct {
+		output       string
+		err          error
+		expectedName string
+		expectErr    bool
+	}{
+		{
+			output:       validOutput,
+			expectedName: interfaces[0].Name,
+		},
+		{
+			output:    invalidOutput,
+			expectErr: true,
+		},
+		{
+			output:    validOutput,
+			err:       errors.New("error"),
+			expectErr: true,
+		},
+	}
+	for _, test := range tests {
+		fcmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				func() ([]byte, error) { return []byte(test.output), test.err },
+			},
+		}
+		fexec := fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{
+				func(cmd string, args ...string) exec.Cmd {
+					return fakeexec.InitFakeCmd(&fcmd, cmd, args...)
+				},
+			},
+			LookPathFunc: func(file string) (string, error) {
+				return fmt.Sprintf("/fake-bin/%s", file), nil
+			},
+		}
+		nsenterArgs := []string{"-t", "123", "-n"}
+		name, err := findPairInterfaceOfContainerInterface(&fexec, "eth0", "123", nsenterArgs)
+		if test.expectErr {
+			if err == nil {
+				t.Errorf("unexpected non-error")
+			}
+		} else {
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+		}
+		if name != test.expectedName {
+			t.Errorf("unexpected name: %s (expected: %s)", name, test.expectedName)
+		}
+	}
+}
+
+func TestSetUpInterfaceNonExistent(t *testing.T) {
+	err := setUpInterface("non-existent")
+	if err == nil {
+		t.Errorf("unexpected non-error")
+	}
+	deviceDir := fmt.Sprintf("%s/%s", sysfsNetPath, "non-existent")
+	if !strings.Contains(fmt.Sprintf("%v", err), deviceDir) {
+		t.Errorf("should have tried to open %s", deviceDir)
+	}
+}
+
+func TestSetUpInterfaceNotBridged(t *testing.T) {
+	err := setUpInterface("lo")
+	if err != nil {
+		if os.IsNotExist(err) {
+			t.Skipf("'lo' device does not exist??? (%v)", err)
+		}
+		t.Errorf("unexpected error: %v", err)
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/BUILD

@@ -0,0 +1,59 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fake_iptables.go",
+        "hostport.go",
+        "hostport_manager.go",
+        "hostport_syncer.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/hostport",
+    deps = [
+        "//pkg/proxy/iptables:go_default_library",
+        "//pkg/util/iptables:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "fake_iptables_test.go",
+        "hostport_manager_test.go",
+        "hostport_syncer_test.go",
+        "hostport_test.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/hostport",
+    library = ":go_default_library",
+    deps = [
+        "//pkg/util/iptables:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//pkg/kubelet/network/hostport/testing:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/fake_iptables.go

@@ -0,0 +1,353 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"bytes"
+	"fmt"
+	"net"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+type fakeChain struct {
+	name  utiliptables.Chain
+	rules []string
+}
+
+type fakeTable struct {
+	name   utiliptables.Table
+	chains map[string]*fakeChain
+}
+
+type fakeIPTables struct {
+	tables        map[string]*fakeTable
+	builtinChains map[string]sets.String
+}
+
+func NewFakeIPTables() *fakeIPTables {
+	return &fakeIPTables{
+		tables: make(map[string]*fakeTable, 0),
+		builtinChains: map[string]sets.String{
+			string(utiliptables.TableFilter): sets.NewString("INPUT", "FORWARD", "OUTPUT"),
+			string(utiliptables.TableNAT):    sets.NewString("PREROUTING", "INPUT", "OUTPUT", "POSTROUTING"),
+			string(utiliptables.TableMangle): sets.NewString("PREROUTING", "INPUT", "FORWARD", "OUTPUT", "POSTROUTING"),
+		},
+	}
+}
+
+func (f *fakeIPTables) GetVersion() (string, error) {
+	return "1.4.21", nil
+}
+
+func (f *fakeIPTables) getTable(tableName utiliptables.Table) (*fakeTable, error) {
+	table, ok := f.tables[string(tableName)]
+	if !ok {
+		return nil, fmt.Errorf("Table %s does not exist", tableName)
+	}
+	return table, nil
+}
+
+func (f *fakeIPTables) getChain(tableName utiliptables.Table, chainName utiliptables.Chain) (*fakeTable, *fakeChain, error) {
+	table, err := f.getTable(tableName)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	chain, ok := table.chains[string(chainName)]
+	if !ok {
+		return table, nil, fmt.Errorf("Chain %s/%s does not exist", tableName, chainName)
+	}
+
+	return table, chain, nil
+}
+
+func (f *fakeIPTables) ensureChain(tableName utiliptables.Table, chainName utiliptables.Chain) (bool, *fakeChain) {
+	table, chain, err := f.getChain(tableName, chainName)
+	if err != nil {
+		// either table or table+chain don't exist yet
+		if table == nil {
+			table = &fakeTable{
+				name:   tableName,
+				chains: make(map[string]*fakeChain),
+			}
+			f.tables[string(tableName)] = table
+		}
+		chain := &fakeChain{
+			name:  chainName,
+			rules: make([]string, 0),
+		}
+		table.chains[string(chainName)] = chain
+		return false, chain
+	}
+	return true, chain
+}
+
+func (f *fakeIPTables) EnsureChain(tableName utiliptables.Table, chainName utiliptables.Chain) (bool, error) {
+	existed, _ := f.ensureChain(tableName, chainName)
+	return existed, nil
+}
+
+func (f *fakeIPTables) FlushChain(tableName utiliptables.Table, chainName utiliptables.Chain) error {
+	_, chain, err := f.getChain(tableName, chainName)
+	if err != nil {
+		return err
+	}
+	chain.rules = make([]string, 0)
+	return nil
+}
+
+func (f *fakeIPTables) DeleteChain(tableName utiliptables.Table, chainName utiliptables.Chain) error {
+	table, _, err := f.getChain(tableName, chainName)
+	if err != nil {
+		return err
+	}
+	delete(table.chains, string(chainName))
+	return nil
+}
+
+// Returns index of rule in array; < 0 if rule is not found
+func findRule(chain *fakeChain, rule string) int {
+	for i, candidate := range chain.rules {
+		if rule == candidate {
+			return i
+		}
+	}
+	return -1
+}
+
+func (f *fakeIPTables) ensureRule(position utiliptables.RulePosition, tableName utiliptables.Table, chainName utiliptables.Chain, rule string) (bool, error) {
+	_, chain, err := f.getChain(tableName, chainName)
+	if err != nil {
+		_, chain = f.ensureChain(tableName, chainName)
+	}
+
+	rule, err = normalizeRule(rule)
+	if err != nil {
+		return false, err
+	}
+	ruleIdx := findRule(chain, rule)
+	if ruleIdx >= 0 {
+		return true, nil
+	}
+
+	if position == utiliptables.Prepend {
+		chain.rules = append([]string{rule}, chain.rules...)
+	} else if position == utiliptables.Append {
+		chain.rules = append(chain.rules, rule)
+	} else {
+		return false, fmt.Errorf("Unknown position argument %q", position)
+	}
+
+	return false, nil
+}
+
+func normalizeRule(rule string) (string, error) {
+	normalized := ""
+	remaining := strings.TrimSpace(rule)
+	for {
+		var end int
+
+		if strings.HasPrefix(remaining, "--to-destination=") {
+			remaining = strings.Replace(remaining, "=", " ", 1)
+		}
+
+		if remaining[0] == '"' {
+			end = strings.Index(remaining[1:], "\"")
+			if end < 0 {
+				return "", fmt.Errorf("Invalid rule syntax: mismatched quotes")
+			}
+			end += 2
+		} else {
+			end = strings.Index(remaining, " ")
+			if end < 0 {
+				end = len(remaining)
+			}
+		}
+		arg := remaining[:end]
+
+		// Normalize un-prefixed IP addresses like iptables does
+		if net.ParseIP(arg) != nil {
+			arg = arg + "/32"
+		}
+
+		if len(normalized) > 0 {
+			normalized += " "
+		}
+		normalized += strings.TrimSpace(arg)
+		if len(remaining) == end {
+			break
+		}
+		remaining = remaining[end+1:]
+	}
+	return normalized, nil
+}
+
+func (f *fakeIPTables) EnsureRule(position utiliptables.RulePosition, tableName utiliptables.Table, chainName utiliptables.Chain, args ...string) (bool, error) {
+	ruleArgs := make([]string, 0)
+	for _, arg := range args {
+		// quote args with internal spaces (like comments)
+		if strings.Index(arg, " ") >= 0 {
+			arg = fmt.Sprintf("\"%s\"", arg)
+		}
+		ruleArgs = append(ruleArgs, arg)
+	}
+	return f.ensureRule(position, tableName, chainName, strings.Join(ruleArgs, " "))
+}
+
+func (f *fakeIPTables) DeleteRule(tableName utiliptables.Table, chainName utiliptables.Chain, args ...string) error {
+	_, chain, err := f.getChain(tableName, chainName)
+	if err == nil {
+		rule := strings.Join(args, " ")
+		ruleIdx := findRule(chain, rule)
+		if ruleIdx < 0 {
+			return nil
+		}
+		chain.rules = append(chain.rules[:ruleIdx], chain.rules[ruleIdx+1:]...)
+	}
+	return nil
+}
+
+func (f *fakeIPTables) IsIpv6() bool {
+	return false
+}
+
+func saveChain(chain *fakeChain, data *bytes.Buffer) {
+	for _, rule := range chain.rules {
+		data.WriteString(fmt.Sprintf("-A %s %s\n", chain.name, rule))
+	}
+}
+
+func (f *fakeIPTables) SaveInto(tableName utiliptables.Table, buffer *bytes.Buffer) error {
+	table, err := f.getTable(tableName)
+	if err != nil {
+		return err
+	}
+
+	buffer.WriteString(fmt.Sprintf("*%s\n", table.name))
+
+	rules := bytes.NewBuffer(nil)
+	for _, chain := range table.chains {
+		buffer.WriteString(fmt.Sprintf(":%s - [0:0]\n", string(chain.name)))
+		saveChain(chain, rules)
+	}
+	buffer.Write(rules.Bytes())
+	buffer.WriteString("COMMIT\n")
+	return nil
+}
+
+func (f *fakeIPTables) restore(restoreTableName utiliptables.Table, data []byte, flush utiliptables.FlushFlag) error {
+	allLines := string(data)
+	buf := bytes.NewBuffer(data)
+	var tableName utiliptables.Table
+	for {
+		line, err := buf.ReadString('\n')
+		if err != nil {
+			break
+		}
+		if line[0] == '#' {
+			continue
+		}
+
+		line = strings.TrimSuffix(line, "\n")
+		if strings.HasPrefix(line, "*") {
+			tableName = utiliptables.Table(line[1:])
+		}
+		if tableName != "" {
+			if restoreTableName != "" && restoreTableName != tableName {
+				continue
+			}
+			if strings.HasPrefix(line, ":") {
+				chainName := utiliptables.Chain(strings.Split(line[1:], " ")[0])
+				if flush == utiliptables.FlushTables {
+					table, chain, _ := f.getChain(tableName, chainName)
+					if chain != nil {
+						delete(table.chains, string(chainName))
+					}
+				}
+				_, _ = f.ensureChain(tableName, chainName)
+				// The --noflush option for iptables-restore doesn't work for user-defined chains, only builtin chains.
+				// We should flush user-defined chains if the chain is not to be deleted
+				if !f.isBuiltinChain(tableName, chainName) && !strings.Contains(allLines, "-X "+string(chainName)) {
+					if err := f.FlushChain(tableName, chainName); err != nil {
+						return err
+					}
+				}
+			} else if strings.HasPrefix(line, "-A") {
+				parts := strings.Split(line, " ")
+				if len(parts) < 3 {
+					return fmt.Errorf("Invalid iptables rule '%s'", line)
+				}
+				chainName := utiliptables.Chain(parts[1])
+				rule := strings.TrimPrefix(line, fmt.Sprintf("-A %s ", chainName))
+				_, err := f.ensureRule(utiliptables.Append, tableName, chainName, rule)
+				if err != nil {
+					return err
+				}
+			} else if strings.HasPrefix(line, "-I") {
+				parts := strings.Split(line, " ")
+				if len(parts) < 3 {
+					return fmt.Errorf("Invalid iptables rule '%s'", line)
+				}
+				chainName := utiliptables.Chain(parts[1])
+				rule := strings.TrimPrefix(line, fmt.Sprintf("-I %s ", chainName))
+				_, err := f.ensureRule(utiliptables.Prepend, tableName, chainName, rule)
+				if err != nil {
+					return err
+				}
+			} else if strings.HasPrefix(line, "-X") {
+				parts := strings.Split(line, " ")
+				if len(parts) < 2 {
+					return fmt.Errorf("Invalid iptables rule '%s'", line)
+				}
+				if err := f.DeleteChain(tableName, utiliptables.Chain(parts[1])); err != nil {
+					return err
+				}
+			} else if line == "COMMIT" {
+				if restoreTableName == tableName {
+					return nil
+				}
+				tableName = ""
+			}
+		}
+	}
+
+	return nil
+}
+
+func (f *fakeIPTables) Restore(tableName utiliptables.Table, data []byte, flush utiliptables.FlushFlag, counters utiliptables.RestoreCountersFlag) error {
+	return f.restore(tableName, data, flush)
+}
+
+func (f *fakeIPTables) RestoreAll(data []byte, flush utiliptables.FlushFlag, counters utiliptables.RestoreCountersFlag) error {
+	return f.restore("", data, flush)
+}
+
+func (f *fakeIPTables) AddReloadFunc(reloadFunc func()) {
+}
+
+func (f *fakeIPTables) Destroy() {
+}
+
+func (f *fakeIPTables) isBuiltinChain(tableName utiliptables.Table, chainName utiliptables.Chain) bool {
+	if builtinChains, ok := f.builtinChains[string(tableName)]; ok && builtinChains.Has(string(chainName)) {
+		return true
+	}
+	return false
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/fake_iptables_test.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+func TestRestoreFlushRules(t *testing.T) {
+	iptables := NewFakeIPTables()
+	rules := [][]string{
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-5N7UH5JAXCVP5UJR"},
+		{"-A", "POSTROUTING", "-m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE"},
+	}
+	natRules := bytes.NewBuffer(nil)
+	writeLine(natRules, "*nat")
+	for _, rule := range rules {
+		_, err := iptables.EnsureChain(utiliptables.TableNAT, utiliptables.Chain(rule[1]))
+		assert.NoError(t, err)
+		_, err = iptables.ensureRule(utiliptables.RulePosition(rule[0]), utiliptables.TableNAT, utiliptables.Chain(rule[1]), rule[2])
+		assert.NoError(t, err)
+
+		writeLine(natRules, utiliptables.MakeChainLine(utiliptables.Chain(rule[1])))
+	}
+	writeLine(natRules, "COMMIT")
+	assert.NoError(t, iptables.Restore(utiliptables.TableNAT, natRules.Bytes(), utiliptables.NoFlushTables, utiliptables.RestoreCounters))
+	natTable, ok := iptables.tables[string(utiliptables.TableNAT)]
+	assert.True(t, ok)
+	// check KUBE-HOSTPORTS chain, should have been cleaned up
+	hostportChain, ok := natTable.chains["KUBE-HOSTPORTS"]
+	assert.True(t, ok)
+	assert.Equal(t, 0, len(hostportChain.rules))
+
+	// check builtin chains, should not been cleaned up
+	postroutingChain, ok := natTable.chains["POSTROUTING"]
+	assert.True(t, ok, string(postroutingChain.name))
+	assert.Equal(t, 1, len(postroutingChain.rules))
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport.go

@@ -0,0 +1,200 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/golang/glog"
+
+	"k8s.io/api/core/v1"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+const (
+	// the hostport chain
+	kubeHostportsChain utiliptables.Chain = "KUBE-HOSTPORTS"
+	// prefix for hostport chains
+	kubeHostportChainPrefix string = "KUBE-HP-"
+)
+
+// PortMapping represents a network port in a container
+type PortMapping struct {
+	Name          string
+	HostPort      int32
+	ContainerPort int32
+	Protocol      v1.Protocol
+	HostIP        string
+}
+
+// PodPortMapping represents a pod's network state and associated container port mappings
+type PodPortMapping struct {
+	Namespace    string
+	Name         string
+	PortMappings []*PortMapping
+	HostNetwork  bool
+	IP           net.IP
+}
+
+// ConstructPodPortMapping creates a PodPortMapping from the ports specified in the pod's
+// containers.
+func ConstructPodPortMapping(pod *v1.Pod, podIP net.IP) *PodPortMapping {
+	portMappings := make([]*PortMapping, 0)
+	for _, c := range pod.Spec.Containers {
+		for _, port := range c.Ports {
+			portMappings = append(portMappings, &PortMapping{
+				Name:          port.Name,
+				HostPort:      port.HostPort,
+				ContainerPort: port.ContainerPort,
+				Protocol:      port.Protocol,
+				HostIP:        port.HostIP,
+			})
+		}
+	}
+
+	return &PodPortMapping{
+		Namespace:    pod.Namespace,
+		Name:         pod.Name,
+		PortMappings: portMappings,
+		HostNetwork:  pod.Spec.HostNetwork,
+		IP:           podIP,
+	}
+}
+
+type hostport struct {
+	port     int32
+	protocol string
+}
+
+type hostportOpener func(*hostport) (closeable, error)
+
+type closeable interface {
+	Close() error
+}
+
+func openLocalPort(hp *hostport) (closeable, error) {
+	// For ports on node IPs, open the actual port and hold it, even though we
+	// use iptables to redirect traffic.
+	// This ensures a) that it's safe to use that port and b) that (a) stays
+	// true.  The risk is that some process on the node (e.g. sshd or kubelet)
+	// is using a port and we give that same port out to a Service.  That would
+	// be bad because iptables would silently claim the traffic but the process
+	// would never know.
+	// NOTE: We should not need to have a real listen()ing socket - bind()
+	// should be enough, but I can't figure out a way to e2e test without
+	// it.  Tools like 'ss' and 'netstat' do not show sockets that are
+	// bind()ed but not listen()ed, and at least the default debian netcat
+	// has no way to avoid about 10 seconds of retries.
+	var socket closeable
+	switch hp.protocol {
+	case "tcp":
+		listener, err := net.Listen("tcp", fmt.Sprintf(":%d", hp.port))
+		if err != nil {
+			return nil, err
+		}
+		socket = listener
+	case "udp":
+		addr, err := net.ResolveUDPAddr("udp", fmt.Sprintf(":%d", hp.port))
+		if err != nil {
+			return nil, err
+		}
+		conn, err := net.ListenUDP("udp", addr)
+		if err != nil {
+			return nil, err
+		}
+		socket = conn
+	default:
+		return nil, fmt.Errorf("unknown protocol %q", hp.protocol)
+	}
+	glog.V(3).Infof("Opened local port %s", hp.String())
+	return socket, nil
+}
+
+// openHostports opens all given hostports using the given hostportOpener
+// If encounter any error, clean up and return the error
+// If all ports are opened successfully, return the hostport and socket mapping
+// TODO: move openHostports and closeHostports into a common struct
+func openHostports(portOpener hostportOpener, podPortMapping *PodPortMapping) (map[hostport]closeable, error) {
+	var retErr error
+	ports := make(map[hostport]closeable)
+	for _, pm := range podPortMapping.PortMappings {
+		if pm.HostPort <= 0 {
+			continue
+		}
+		hp := portMappingToHostport(pm)
+		socket, err := portOpener(&hp)
+		if err != nil {
+			retErr = fmt.Errorf("cannot open hostport %d for pod %s: %v", pm.HostPort, getPodFullName(podPortMapping), err)
+			break
+		}
+		ports[hp] = socket
+	}
+
+	// If encounter any error, close all hostports that just got opened.
+	if retErr != nil {
+		for hp, socket := range ports {
+			if err := socket.Close(); err != nil {
+				glog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podPortMapping), err)
+			}
+		}
+		return nil, retErr
+	}
+	return ports, nil
+}
+
+// portMappingToHostport creates hostport structure based on input portmapping
+func portMappingToHostport(portMapping *PortMapping) hostport {
+	return hostport{
+		port:     portMapping.HostPort,
+		protocol: strings.ToLower(string(portMapping.Protocol)),
+	}
+}
+
+// ensureKubeHostportChains ensures the KUBE-HOSTPORTS chain is setup correctly
+func ensureKubeHostportChains(iptables utiliptables.Interface, natInterfaceName string) error {
+	glog.V(4).Info("Ensuring kubelet hostport chains")
+	// Ensure kubeHostportChain
+	if _, err := iptables.EnsureChain(utiliptables.TableNAT, kubeHostportsChain); err != nil {
+		return fmt.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, kubeHostportsChain, err)
+	}
+	tableChainsNeedJumpServices := []struct {
+		table utiliptables.Table
+		chain utiliptables.Chain
+	}{
+		{utiliptables.TableNAT, utiliptables.ChainOutput},
+		{utiliptables.TableNAT, utiliptables.ChainPrerouting},
+	}
+	args := []string{"-m", "comment", "--comment", "kube hostport portals",
+		"-m", "addrtype", "--dst-type", "LOCAL",
+		"-j", string(kubeHostportsChain)}
+	for _, tc := range tableChainsNeedJumpServices {
+		// KUBE-HOSTPORTS chain needs to be appended to the system chains.
+		// This ensures KUBE-SERVICES chain gets processed first.
+		// Since rules in KUBE-HOSTPORTS chain matches broader cases, allow the more specific rules to be processed first.
+		if _, err := iptables.EnsureRule(utiliptables.Append, tc.table, tc.chain, args...); err != nil {
+			return fmt.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", tc.table, tc.chain, kubeHostportsChain, err)
+		}
+	}
+	// Need to SNAT traffic from localhost
+	args = []string{"-m", "comment", "--comment", "SNAT for localhost access to hostports", "-o", natInterfaceName, "-s", "127.0.0.0/8", "-j", "MASQUERADE"}
+	if _, err := iptables.EnsureRule(utiliptables.Append, utiliptables.TableNAT, utiliptables.ChainPostrouting, args...); err != nil {
+		return fmt.Errorf("Failed to ensure that %s chain %s jumps to MASQUERADE: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, err)
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_manager.go

@@ -0,0 +1,334 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base32"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/golang/glog"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+// HostPortManager is an interface for adding and removing hostport for a given pod sandbox.
+type HostPortManager interface {
+	// Add implements port mappings.
+	// id should be a unique identifier for a pod, e.g. podSandboxID.
+	// podPortMapping is the associated port mapping information for the pod.
+	// natInterfaceName is the interface that localhost used to talk to the given pod.
+	Add(id string, podPortMapping *PodPortMapping, natInterfaceName string) error
+	// Remove cleans up matching port mappings
+	// Remove must be able to clean up port mappings without pod IP
+	Remove(id string, podPortMapping *PodPortMapping) error
+}
+
+type hostportManager struct {
+	hostPortMap map[hostport]closeable
+	iptables    utiliptables.Interface
+	portOpener  hostportOpener
+	mu          sync.Mutex
+}
+
+func NewHostportManager(iptables utiliptables.Interface) HostPortManager {
+	return &hostportManager{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    iptables,
+		portOpener:  openLocalPort,
+	}
+}
+
+func (hm *hostportManager) Add(id string, podPortMapping *PodPortMapping, natInterfaceName string) (err error) {
+	if podPortMapping == nil || podPortMapping.HostNetwork {
+		return nil
+	}
+	podFullName := getPodFullName(podPortMapping)
+
+	// skip if there is no hostport needed
+	hostportMappings := gatherHostportMappings(podPortMapping)
+	if len(hostportMappings) == 0 {
+		return nil
+	}
+
+	if podPortMapping.IP.To4() == nil {
+		return fmt.Errorf("invalid or missing IP of pod %s", podFullName)
+	}
+	podIP := podPortMapping.IP.String()
+
+	if err = ensureKubeHostportChains(hm.iptables, natInterfaceName); err != nil {
+		return err
+	}
+
+	// Ensure atomicity for port opening and iptables operations
+	hm.mu.Lock()
+	defer hm.mu.Unlock()
+
+	// try to open hostports
+	ports, err := openHostports(hm.portOpener, podPortMapping)
+	if err != nil {
+		return err
+	}
+	for hostport, socket := range ports {
+		hm.hostPortMap[hostport] = socket
+	}
+
+	natChains := bytes.NewBuffer(nil)
+	natRules := bytes.NewBuffer(nil)
+	writeLine(natChains, "*nat")
+
+	existingChains, existingRules, err := getExistingHostportIPTablesRules(hm.iptables)
+	if err != nil {
+		// clean up opened host port if encounter any error
+		return utilerrors.NewAggregate([]error{err, hm.closeHostports(hostportMappings)})
+	}
+
+	newChains := []utiliptables.Chain{}
+	for _, pm := range hostportMappings {
+		protocol := strings.ToLower(string(pm.Protocol))
+		chain := getHostportChain(id, pm)
+		newChains = append(newChains, chain)
+
+		// Add new hostport chain
+		writeLine(natChains, utiliptables.MakeChainLine(chain))
+
+		// Prepend the new chain to KUBE-HOSTPORTS
+		// This avoids any leaking iptables rule that takes up the same port
+		writeLine(natRules, "-I", string(kubeHostportsChain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
+			"-m", protocol, "-p", protocol, "--dport", fmt.Sprintf("%d", pm.HostPort),
+			"-j", string(chain),
+		)
+
+		// SNAT if the traffic comes from the pod itself
+		writeLine(natRules, "-A", string(chain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
+			"-s", podIP,
+			"-j", string(iptablesproxy.KubeMarkMasqChain))
+
+		// DNAT to the podIP:containerPort
+		writeLine(natRules, "-A", string(chain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, podFullName, pm.HostPort),
+			"-m", protocol, "-p", protocol,
+			"-j", "DNAT", fmt.Sprintf("--to-destination=%s:%d", podIP, pm.ContainerPort))
+	}
+
+	// getHostportChain should be able to provide unique hostport chain name using hash
+	// if there is a chain conflict or multiple Adds have been triggered for a single pod,
+	// filtering should be able to avoid further problem
+	filterChains(existingChains, newChains)
+	existingRules = filterRules(existingRules, newChains)
+
+	for _, chain := range existingChains {
+		writeLine(natChains, chain)
+	}
+	for _, rule := range existingRules {
+		writeLine(natRules, rule)
+	}
+	writeLine(natRules, "COMMIT")
+
+	if err = hm.syncIPTables(append(natChains.Bytes(), natRules.Bytes()...)); err != nil {
+		// clean up opened host port if encounter any error
+		return utilerrors.NewAggregate([]error{err, hm.closeHostports(hostportMappings)})
+	}
+	return nil
+}
+
+func (hm *hostportManager) Remove(id string, podPortMapping *PodPortMapping) (err error) {
+	if podPortMapping == nil || podPortMapping.HostNetwork {
+		return nil
+	}
+
+	hostportMappings := gatherHostportMappings(podPortMapping)
+	if len(hostportMappings) <= 0 {
+		return nil
+	}
+
+	// Ensure atomicity for port closing and iptables operations
+	hm.mu.Lock()
+	defer hm.mu.Unlock()
+
+	var existingChains map[utiliptables.Chain]string
+	var existingRules []string
+	existingChains, existingRules, err = getExistingHostportIPTablesRules(hm.iptables)
+	if err != nil {
+		return err
+	}
+
+	// Gather target hostport chains for removal
+	chainsToRemove := []utiliptables.Chain{}
+	for _, pm := range hostportMappings {
+		chainsToRemove = append(chainsToRemove, getHostportChain(id, pm))
+		// TODO remove this after release 1.9, please refer https://github.com/kubernetes/kubernetes/pull/55153
+		chainsToRemove = append(chainsToRemove, getBuggyHostportChain(id, pm))
+	}
+
+	// remove rules that consists of target chains
+	remainingRules := filterRules(existingRules, chainsToRemove)
+
+	// gather target hostport chains that exists in iptables-save result
+	existingChainsToRemove := []utiliptables.Chain{}
+	for _, chain := range chainsToRemove {
+		if _, ok := existingChains[chain]; ok {
+			existingChainsToRemove = append(existingChainsToRemove, chain)
+		}
+	}
+
+	natChains := bytes.NewBuffer(nil)
+	natRules := bytes.NewBuffer(nil)
+	writeLine(natChains, "*nat")
+	for _, chain := range existingChains {
+		writeLine(natChains, chain)
+	}
+	for _, rule := range remainingRules {
+		writeLine(natRules, rule)
+	}
+	for _, chain := range existingChainsToRemove {
+		writeLine(natRules, "-X", string(chain))
+	}
+	writeLine(natRules, "COMMIT")
+
+	if err = hm.syncIPTables(append(natChains.Bytes(), natRules.Bytes()...)); err != nil {
+		return err
+	}
+
+	// clean up opened pod host ports
+	return hm.closeHostports(hostportMappings)
+}
+
+// syncIPTables executes iptables-restore with given lines
+func (hm *hostportManager) syncIPTables(lines []byte) error {
+	glog.V(3).Infof("Restoring iptables rules: %s", lines)
+	err := hm.iptables.RestoreAll(lines, utiliptables.NoFlushTables, utiliptables.RestoreCounters)
+	if err != nil {
+		return fmt.Errorf("Failed to execute iptables-restore: %v", err)
+	}
+	return nil
+}
+
+// closeHostports tries to close all the listed host ports
+// TODO: move closeHostports and openHostports into a common struct
+func (hm *hostportManager) closeHostports(hostportMappings []*PortMapping) error {
+	errList := []error{}
+	for _, pm := range hostportMappings {
+		hp := portMappingToHostport(pm)
+		if socket, ok := hm.hostPortMap[hp]; ok {
+			glog.V(2).Infof("Closing host port %s", hp.String())
+			if err := socket.Close(); err != nil {
+				errList = append(errList, fmt.Errorf("failed to close host port %s: %v", hp.String(), err))
+				continue
+			}
+			delete(hm.hostPortMap, hp)
+		}
+	}
+	return utilerrors.NewAggregate(errList)
+}
+
+// getHostportChain takes id, hostport and protocol for a pod and returns associated iptables chain.
+// This is computed by hashing (sha256) then encoding to base32 and truncating with the prefix
+// "KUBE-HP-". We do this because IPTables Chain Names must be <= 28 chars long, and the longer
+// they are the harder they are to read.
+// WARNING: Please do not change this function. Otherwise, HostportManager may not be able to
+// identify existing iptables chains.
+func getHostportChain(id string, pm *PortMapping) utiliptables.Chain {
+	hash := sha256.Sum256([]byte(id + strconv.Itoa(int(pm.HostPort)) + string(pm.Protocol)))
+	encoded := base32.StdEncoding.EncodeToString(hash[:])
+	return utiliptables.Chain(kubeHostportChainPrefix + encoded[:16])
+}
+
+// This bugy func does bad conversion on HostPort from int32 to string.
+// It may generates same chain names for different ports of the same pod, e.g. port 57119/55429/56833.
+// `getHostportChain` fixed this bug. In order to cleanup the legacy chains/rules, it is temporarily left.
+// TODO remove this after release 1.9, please refer https://github.com/kubernetes/kubernetes/pull/55153
+func getBuggyHostportChain(id string, pm *PortMapping) utiliptables.Chain {
+	hash := sha256.Sum256([]byte(id + string(pm.HostPort) + string(pm.Protocol)))
+	encoded := base32.StdEncoding.EncodeToString(hash[:])
+	return utiliptables.Chain(kubeHostportChainPrefix + encoded[:16])
+}
+
+// gatherHostportMappings returns all the PortMappings which has hostport for a pod
+func gatherHostportMappings(podPortMapping *PodPortMapping) []*PortMapping {
+	mappings := []*PortMapping{}
+	for _, pm := range podPortMapping.PortMappings {
+		if pm.HostPort <= 0 {
+			continue
+		}
+		mappings = append(mappings, pm)
+	}
+	return mappings
+}
+
+// getExistingHostportIPTablesRules retrieves raw data from iptables-save, parse it,
+// return all the hostport related chains and rules
+func getExistingHostportIPTablesRules(iptables utiliptables.Interface) (map[utiliptables.Chain]string, []string, error) {
+	iptablesData := bytes.NewBuffer(nil)
+	err := iptables.SaveInto(utiliptables.TableNAT, iptablesData)
+	if err != nil { // if we failed to get any rules
+		return nil, nil, fmt.Errorf("failed to execute iptables-save: %v", err)
+	}
+	existingNATChains := utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes())
+
+	existingHostportChains := make(map[utiliptables.Chain]string)
+	existingHostportRules := []string{}
+
+	for chain := range existingNATChains {
+		if strings.HasPrefix(string(chain), string(kubeHostportsChain)) || strings.HasPrefix(string(chain), kubeHostportChainPrefix) {
+			existingHostportChains[chain] = existingNATChains[chain]
+		}
+	}
+
+	for _, line := range strings.Split(string(iptablesData.Bytes()), "\n") {
+		if strings.HasPrefix(line, fmt.Sprintf("-A %s", kubeHostportChainPrefix)) ||
+			strings.HasPrefix(line, fmt.Sprintf("-A %s", string(kubeHostportsChain))) {
+			existingHostportRules = append(existingHostportRules, line)
+		}
+	}
+	return existingHostportChains, existingHostportRules, nil
+}
+
+// filterRules filters input rules with input chains. Rules that did not involve any filter chain will be returned.
+// The order of the input rules is important and is preserved.
+func filterRules(rules []string, filters []utiliptables.Chain) []string {
+	filtered := []string{}
+	for _, rule := range rules {
+		skip := false
+		for _, filter := range filters {
+			if strings.Contains(rule, string(filter)) {
+				skip = true
+				break
+			}
+		}
+		if !skip {
+			filtered = append(filtered, rule)
+		}
+	}
+	return filtered
+}
+
+// filterChains deletes all entries of filter chains from chain map
+func filterChains(chains map[utiliptables.Chain]string, filterChains []utiliptables.Chain) {
+	for _, chain := range filterChains {
+		if _, ok := chains[chain]; ok {
+			delete(chains, chain)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_manager_test.go

@@ -0,0 +1,289 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"bytes"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+func NewFakeHostportManager() HostPortManager {
+	return &hostportManager{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    NewFakeIPTables(),
+		portOpener:  NewFakeSocketManager().openFakeSocket,
+	}
+}
+
+func TestHostportManager(t *testing.T) {
+	iptables := NewFakeIPTables()
+	portOpener := NewFakeSocketManager()
+	manager := &hostportManager{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    iptables,
+		portOpener:  portOpener.openFakeSocket,
+	}
+
+	testCases := []struct {
+		mapping     *PodPortMapping
+		expectError bool
+	}{
+		{
+			mapping: &PodPortMapping{
+				Name:        "pod1",
+				Namespace:   "ns1",
+				IP:          net.ParseIP("10.1.1.2"),
+				HostNetwork: false,
+				PortMappings: []*PortMapping{
+					{
+						HostPort:      8080,
+						ContainerPort: 80,
+						Protocol:      v1.ProtocolTCP,
+					},
+					{
+						HostPort:      8081,
+						ContainerPort: 81,
+						Protocol:      v1.ProtocolUDP,
+					},
+				},
+			},
+			expectError: false,
+		},
+		{
+			mapping: &PodPortMapping{
+				Name:        "pod2",
+				Namespace:   "ns1",
+				IP:          net.ParseIP("10.1.1.3"),
+				HostNetwork: false,
+				PortMappings: []*PortMapping{
+					{
+						HostPort:      8082,
+						ContainerPort: 80,
+						Protocol:      v1.ProtocolTCP,
+					},
+					{
+						HostPort:      8081,
+						ContainerPort: 81,
+						Protocol:      v1.ProtocolUDP,
+					},
+				},
+			},
+			expectError: true,
+		},
+		{
+			mapping: &PodPortMapping{
+				Name:        "pod3",
+				Namespace:   "ns1",
+				IP:          net.ParseIP("10.1.1.4"),
+				HostNetwork: false,
+				PortMappings: []*PortMapping{
+					{
+						HostPort:      8443,
+						ContainerPort: 443,
+						Protocol:      v1.ProtocolTCP,
+					},
+				},
+			},
+			expectError: false,
+		},
+	}
+
+	// Add Hostports
+	for _, tc := range testCases {
+		err := manager.Add("id", tc.mapping, "cbr0")
+		if tc.expectError {
+			assert.Error(t, err)
+			continue
+		}
+		assert.NoError(t, err)
+	}
+
+	// Check port opened
+	expectedPorts := []hostport{{8080, "tcp"}, {8081, "udp"}, {8443, "tcp"}}
+	openedPorts := make(map[hostport]bool)
+	for hp, port := range portOpener.mem {
+		if !port.closed {
+			openedPorts[hp] = true
+		}
+	}
+	assert.EqualValues(t, len(openedPorts), len(expectedPorts))
+	for _, hp := range expectedPorts {
+		_, ok := openedPorts[hp]
+		assert.EqualValues(t, true, ok)
+	}
+
+	// Check Iptables-save result after adding hostports
+	raw := bytes.NewBuffer(nil)
+	err := iptables.SaveInto(utiliptables.TableNAT, raw)
+	assert.NoError(t, err)
+
+	lines := strings.Split(string(raw.Bytes()), "\n")
+	expectedLines := map[string]bool{
+		`*nat`: true,
+		`:KUBE-HOSTPORTS - [0:0]`:                                                                                                         true,
+		`:OUTPUT - [0:0]`:                                                                                                                 true,
+		`:PREROUTING - [0:0]`:                                                                                                             true,
+		`:POSTROUTING - [0:0]`:                                                                                                            true,
+		`:KUBE-HP-IJHALPHTORMHHPPK - [0:0]`:                                                                                               true,
+		`:KUBE-HP-63UPIDJXVRSZGSUZ - [0:0]`:                                                                                               true,
+		`:KUBE-HP-WFBOALXEP42XEMJK - [0:0]`:                                                                                               true,
+		"-A KUBE-HOSTPORTS -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-WFBOALXEP42XEMJK":        true,
+		"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp --dport 8081 -j KUBE-HP-63UPIDJXVRSZGSUZ":        true,
+		"-A KUBE-HOSTPORTS -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp --dport 8080 -j KUBE-HP-IJHALPHTORMHHPPK":        true,
+		"-A OUTPUT -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS":                         true,
+		"-A PREROUTING -m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS":                     true,
+		"-A POSTROUTING -m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE":             true,
+		"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ":                    true,
+		"-A KUBE-HP-IJHALPHTORMHHPPK -m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80":  true,
+		"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ":                    true,
+		"-A KUBE-HP-63UPIDJXVRSZGSUZ -m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81":  true,
+		"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ":                    true,
+		"-A KUBE-HP-WFBOALXEP42XEMJK -m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443": true,
+		`COMMIT`: true,
+	}
+	for _, line := range lines {
+		if len(strings.TrimSpace(line)) > 0 {
+			_, ok := expectedLines[strings.TrimSpace(line)]
+			assert.EqualValues(t, true, ok)
+		}
+	}
+
+	// Remove all added hostports
+	for _, tc := range testCases {
+		if !tc.expectError {
+			err := manager.Remove("id", tc.mapping)
+			assert.NoError(t, err)
+		}
+	}
+
+	// Check Iptables-save result after deleting hostports
+	raw.Reset()
+	err = iptables.SaveInto(utiliptables.TableNAT, raw)
+	assert.NoError(t, err)
+	lines = strings.Split(string(raw.Bytes()), "\n")
+	remainingChains := make(map[string]bool)
+	for _, line := range lines {
+		if strings.HasPrefix(line, ":") {
+			remainingChains[strings.TrimSpace(line)] = true
+		}
+	}
+	expectDeletedChains := []string{"KUBE-HP-4YVONL46AKYWSKS3", "KUBE-HP-7THKRFSEH4GIIXK7", "KUBE-HP-5N7UH5JAXCVP5UJR"}
+	for _, chain := range expectDeletedChains {
+		_, ok := remainingChains[chain]
+		assert.EqualValues(t, false, ok)
+	}
+
+	// check if all ports are closed
+	for _, port := range portOpener.mem {
+		assert.EqualValues(t, true, port.closed)
+	}
+}
+
+func TestGetHostportChain(t *testing.T) {
+	m := make(map[string]int)
+	chain := getHostportChain("testrdma-2", &PortMapping{HostPort: 57119, Protocol: "TCP", ContainerPort: 57119})
+	m[string(chain)] = 1
+	chain = getHostportChain("testrdma-2", &PortMapping{HostPort: 55429, Protocol: "TCP", ContainerPort: 55429})
+	m[string(chain)] = 1
+	chain = getHostportChain("testrdma-2", &PortMapping{HostPort: 56833, Protocol: "TCP", ContainerPort: 56833})
+	m[string(chain)] = 1
+	if len(m) != 3 {
+		t.Fatal(m)
+	}
+}
+
+func TestHostPortManagerRemoveLegacyRules(t *testing.T) {
+	iptables := NewFakeIPTables()
+	legacyRules := [][]string{
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-5N7UH5JAXCVP5UJR"},
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp --dport 8081 -j KUBE-HP-7THKRFSEH4GIIXK7"},
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp --dport 8080 -j KUBE-HP-4YVONL46AKYWSKS3"},
+		{"-A", "OUTPUT", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+		{"-A", "PREROUTING", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+		{"-A", "POSTROUTING", "-m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE"},
+		{"-A", "KUBE-HP-4YVONL46AKYWSKS3", "-m comment --comment \"pod1_ns1 hostport 8080\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-4YVONL46AKYWSKS3", "-m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80"},
+		{"-A", "KUBE-HP-7THKRFSEH4GIIXK7", "-m comment --comment \"pod1_ns1 hostport 8081\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-7THKRFSEH4GIIXK7", "-m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81"},
+		{"-A", "KUBE-HP-5N7UH5JAXCVP5UJR", "-m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-5N7UH5JAXCVP5UJR", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443"},
+	}
+	for _, rule := range legacyRules {
+		_, err := iptables.EnsureChain(utiliptables.TableNAT, utiliptables.Chain(rule[1]))
+		assert.NoError(t, err)
+		_, err = iptables.ensureRule(utiliptables.RulePosition(rule[0]), utiliptables.TableNAT, utiliptables.Chain(rule[1]), rule[2])
+		assert.NoError(t, err)
+	}
+	portOpener := NewFakeSocketManager()
+	manager := &hostportManager{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    iptables,
+		portOpener:  portOpener.openFakeSocket,
+	}
+	err := manager.Remove("id", &PodPortMapping{
+		Name:        "pod1",
+		Namespace:   "ns1",
+		IP:          net.ParseIP("10.1.1.2"),
+		HostNetwork: false,
+		PortMappings: []*PortMapping{
+			{
+				HostPort:      8080,
+				ContainerPort: 80,
+				Protocol:      v1.ProtocolTCP,
+			},
+			{
+				HostPort:      8081,
+				ContainerPort: 81,
+				Protocol:      v1.ProtocolUDP,
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	err = manager.Remove("id", &PodPortMapping{
+		Name:        "pod3",
+		Namespace:   "ns1",
+		IP:          net.ParseIP("10.1.1.4"),
+		HostNetwork: false,
+		PortMappings: []*PortMapping{
+			{
+				HostPort:      8443,
+				ContainerPort: 443,
+				Protocol:      v1.ProtocolTCP,
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	natTable, ok := iptables.tables[string(utiliptables.TableNAT)]
+	assert.True(t, ok)
+	// check KUBE-HOSTPORTS chain should be cleaned up
+	hostportChain, ok := natTable.chains["KUBE-HOSTPORTS"]
+	assert.True(t, ok, string(hostportChain.name))
+	assert.Equal(t, 0, len(hostportChain.rules), "%v", hostportChain.rules)
+	// check KUBE-HP-* chains should be deleted
+	for _, name := range []string{"KUBE-HP-4YVONL46AKYWSKS3", "KUBE-HP-7THKRFSEH4GIIXK7", "KUBE-HP-5N7UH5JAXCVP5UJR"} {
+		_, ok := natTable.chains[name]
+		assert.False(t, ok)
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_syncer.go

@@ -0,0 +1,304 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base32"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+
+	iptablesproxy "k8s.io/kubernetes/pkg/proxy/iptables"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+// HostportSyncer takes a list of PodPortMappings and implements hostport all at once
+type HostportSyncer interface {
+	// SyncHostports gathers all hostports on node and setup iptables rules to enable them.
+	// On each invocation existing ports are synced and stale rules are deleted.
+	SyncHostports(natInterfaceName string, activePodPortMappings []*PodPortMapping) error
+	// OpenPodHostportsAndSync opens hostports for a new PodPortMapping, gathers all hostports on
+	// node, sets up iptables rules enable them. On each invocation existing ports are synced and stale rules are deleted.
+	// 'newPortMapping' must also be present in 'activePodPortMappings'.
+	OpenPodHostportsAndSync(newPortMapping *PodPortMapping, natInterfaceName string, activePodPortMappings []*PodPortMapping) error
+}
+
+type hostportSyncer struct {
+	hostPortMap map[hostport]closeable
+	iptables    utiliptables.Interface
+	portOpener  hostportOpener
+}
+
+func NewHostportSyncer(iptables utiliptables.Interface) HostportSyncer {
+	return &hostportSyncer{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    iptables,
+		portOpener:  openLocalPort,
+	}
+}
+
+type targetPod struct {
+	podFullName string
+	podIP       string
+}
+
+func (hp *hostport) String() string {
+	return fmt.Sprintf("%s:%d", hp.protocol, hp.port)
+}
+
+// openHostports opens all hostport for pod and returns the map of hostport and socket
+func (h *hostportSyncer) openHostports(podHostportMapping *PodPortMapping) error {
+	var retErr error
+	ports := make(map[hostport]closeable)
+	for _, port := range podHostportMapping.PortMappings {
+		if port.HostPort <= 0 {
+			// Assume hostport is not specified in this portmapping. So skip
+			continue
+		}
+		hp := hostport{
+			port:     port.HostPort,
+			protocol: strings.ToLower(string(port.Protocol)),
+		}
+		socket, err := h.portOpener(&hp)
+		if err != nil {
+			retErr = fmt.Errorf("cannot open hostport %d for pod %s: %v", port.HostPort, getPodFullName(podHostportMapping), err)
+			break
+		}
+		ports[hp] = socket
+	}
+
+	// If encounter any error, close all hostports that just got opened.
+	if retErr != nil {
+		for hp, socket := range ports {
+			if err := socket.Close(); err != nil {
+				glog.Errorf("Cannot clean up hostport %d for pod %s: %v", hp.port, getPodFullName(podHostportMapping), err)
+			}
+		}
+		return retErr
+	}
+
+	for hostPort, socket := range ports {
+		h.hostPortMap[hostPort] = socket
+	}
+
+	return nil
+}
+
+func getPodFullName(pod *PodPortMapping) string {
+	// Use underscore as the delimiter because it is not allowed in pod name
+	// (DNS subdomain format), while allowed in the container name format.
+	return pod.Name + "_" + pod.Namespace
+}
+
+// gatherAllHostports returns all hostports that should be presented on node,
+// given the list of pods running on that node and ignoring host network
+// pods (which don't need hostport <-> container port mapping).
+func gatherAllHostports(activePodPortMappings []*PodPortMapping) (map[*PortMapping]targetPod, error) {
+	podHostportMap := make(map[*PortMapping]targetPod)
+	for _, pm := range activePodPortMappings {
+		if pm.IP.To4() == nil {
+			return nil, fmt.Errorf("Invalid or missing pod %s IP", getPodFullName(pm))
+		}
+		// should not handle hostports for hostnetwork pods
+		if pm.HostNetwork {
+			continue
+		}
+
+		for _, port := range pm.PortMappings {
+			if port.HostPort != 0 {
+				podHostportMap[port] = targetPod{podFullName: getPodFullName(pm), podIP: pm.IP.String()}
+			}
+		}
+	}
+	return podHostportMap, nil
+}
+
+// Join all words with spaces, terminate with newline and write to buf.
+func writeLine(buf *bytes.Buffer, words ...string) {
+	buf.WriteString(strings.Join(words, " ") + "\n")
+}
+
+//hostportChainName takes containerPort for a pod and returns associated iptables chain.
+// This is computed by hashing (sha256)
+// then encoding to base32 and truncating with the prefix "KUBE-SVC-".  We do
+// this because IPTables Chain Names must be <= 28 chars long, and the longer
+// they are the harder they are to read.
+func hostportChainName(pm *PortMapping, podFullName string) utiliptables.Chain {
+	hash := sha256.Sum256([]byte(strconv.Itoa(int(pm.HostPort)) + string(pm.Protocol) + podFullName))
+	encoded := base32.StdEncoding.EncodeToString(hash[:])
+	return utiliptables.Chain(kubeHostportChainPrefix + encoded[:16])
+}
+
+// OpenPodHostportsAndSync opens hostports for a new PodPortMapping, gathers all hostports on
+// node, sets up iptables rules enable them. And finally clean up stale hostports.
+// 'newPortMapping' must also be present in 'activePodPortMappings'.
+func (h *hostportSyncer) OpenPodHostportsAndSync(newPortMapping *PodPortMapping, natInterfaceName string, activePodPortMappings []*PodPortMapping) error {
+	// try to open pod host port if specified
+	if err := h.openHostports(newPortMapping); err != nil {
+		return err
+	}
+
+	// Add the new pod to active pods if it's not present.
+	var found bool
+	for _, pm := range activePodPortMappings {
+		if pm.Namespace == newPortMapping.Namespace && pm.Name == newPortMapping.Name {
+			found = true
+			break
+		}
+	}
+	if !found {
+		activePodPortMappings = append(activePodPortMappings, newPortMapping)
+	}
+
+	return h.SyncHostports(natInterfaceName, activePodPortMappings)
+}
+
+// SyncHostports gathers all hostports on node and setup iptables rules enable them. And finally clean up stale hostports
+func (h *hostportSyncer) SyncHostports(natInterfaceName string, activePodPortMappings []*PodPortMapping) error {
+	start := time.Now()
+	defer func() {
+		glog.V(4).Infof("syncHostportsRules took %v", time.Since(start))
+	}()
+
+	hostportPodMap, err := gatherAllHostports(activePodPortMappings)
+	if err != nil {
+		return err
+	}
+
+	// Ensure KUBE-HOSTPORTS chains
+	ensureKubeHostportChains(h.iptables, natInterfaceName)
+
+	// Get iptables-save output so we can check for existing chains and rules.
+	// This will be a map of chain name to chain with rules as stored in iptables-save/iptables-restore
+	existingNATChains := make(map[utiliptables.Chain]string)
+	iptablesData := bytes.NewBuffer(nil)
+	err = h.iptables.SaveInto(utiliptables.TableNAT, iptablesData)
+	if err != nil { // if we failed to get any rules
+		glog.Errorf("Failed to execute iptables-save, syncing all rules: %v", err)
+	} else { // otherwise parse the output
+		existingNATChains = utiliptables.GetChainLines(utiliptables.TableNAT, iptablesData.Bytes())
+	}
+
+	natChains := bytes.NewBuffer(nil)
+	natRules := bytes.NewBuffer(nil)
+	writeLine(natChains, "*nat")
+	// Make sure we keep stats for the top-level chains, if they existed
+	// (which most should have because we created them above).
+	if chain, ok := existingNATChains[kubeHostportsChain]; ok {
+		writeLine(natChains, chain)
+	} else {
+		writeLine(natChains, utiliptables.MakeChainLine(kubeHostportsChain))
+	}
+
+	// Accumulate NAT chains to keep.
+	activeNATChains := map[utiliptables.Chain]bool{} // use a map as a set
+
+	for port, target := range hostportPodMap {
+		protocol := strings.ToLower(string(port.Protocol))
+		hostportChain := hostportChainName(port, target.podFullName)
+		if chain, ok := existingNATChains[hostportChain]; ok {
+			writeLine(natChains, chain)
+		} else {
+			writeLine(natChains, utiliptables.MakeChainLine(hostportChain))
+		}
+
+		activeNATChains[hostportChain] = true
+
+		// Redirect to hostport chain
+		args := []string{
+			"-A", string(kubeHostportsChain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, target.podFullName, port.HostPort),
+			"-m", protocol, "-p", protocol,
+			"--dport", fmt.Sprintf("%d", port.HostPort),
+			"-j", string(hostportChain),
+		}
+		writeLine(natRules, args...)
+
+		// Assuming kubelet is syncing iptables KUBE-MARK-MASQ chain
+		// If the request comes from the pod that is serving the hostport, then SNAT
+		args = []string{
+			"-A", string(hostportChain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, target.podFullName, port.HostPort),
+			"-s", target.podIP, "-j", string(iptablesproxy.KubeMarkMasqChain),
+		}
+		writeLine(natRules, args...)
+
+		// Create hostport chain to DNAT traffic to final destination
+		// IPTables will maintained the stats for this chain
+		args = []string{
+			"-A", string(hostportChain),
+			"-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, target.podFullName, port.HostPort),
+			"-m", protocol, "-p", protocol,
+			"-j", "DNAT", fmt.Sprintf("--to-destination=%s:%d", target.podIP, port.ContainerPort),
+		}
+		writeLine(natRules, args...)
+	}
+
+	// Delete chains no longer in use.
+	for chain := range existingNATChains {
+		if !activeNATChains[chain] {
+			chainString := string(chain)
+			if !strings.HasPrefix(chainString, kubeHostportChainPrefix) {
+				// Ignore chains that aren't ours.
+				continue
+			}
+			// We must (as per iptables) write a chain-line for it, which has
+			// the nice effect of flushing the chain.  Then we can remove the
+			// chain.
+			writeLine(natChains, existingNATChains[chain])
+			writeLine(natRules, "-X", chainString)
+		}
+	}
+	writeLine(natRules, "COMMIT")
+
+	natLines := append(natChains.Bytes(), natRules.Bytes()...)
+	glog.V(3).Infof("Restoring iptables rules: %s", natLines)
+	err = h.iptables.RestoreAll(natLines, utiliptables.NoFlushTables, utiliptables.RestoreCounters)
+	if err != nil {
+		return fmt.Errorf("Failed to execute iptables-restore: %v", err)
+	}
+
+	h.cleanupHostportMap(hostportPodMap)
+	return nil
+}
+
+// cleanupHostportMap closes obsolete hostports
+func (h *hostportSyncer) cleanupHostportMap(containerPortMap map[*PortMapping]targetPod) {
+	// compute hostports that are supposed to be open
+	currentHostports := make(map[hostport]bool)
+	for containerPort := range containerPortMap {
+		hp := hostport{
+			port:     containerPort.HostPort,
+			protocol: strings.ToLower(string(containerPort.Protocol)),
+		}
+		currentHostports[hp] = true
+	}
+
+	// close and delete obsolete hostports
+	for hp, socket := range h.hostPortMap {
+		if _, ok := currentHostports[hp]; !ok {
+			socket.Close()
+			glog.V(3).Infof("Closed local port %s", hp.String())
+			delete(h.hostPortMap, hp)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_syncer_test.go

@@ -0,0 +1,312 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+type ruleMatch struct {
+	hostport int
+	chain    string
+	match    string
+}
+
+func TestOpenPodHostports(t *testing.T) {
+	fakeIPTables := NewFakeIPTables()
+	fakeOpener := NewFakeSocketManager()
+
+	h := &hostportSyncer{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    fakeIPTables,
+		portOpener:  fakeOpener.openFakeSocket,
+	}
+
+	tests := []struct {
+		mapping *PodPortMapping
+		matches []*ruleMatch
+	}{
+		// New pod that we are going to add
+		{
+			&PodPortMapping{
+				Name:        "test-pod",
+				Namespace:   v1.NamespaceDefault,
+				IP:          net.ParseIP("10.1.1.2"),
+				HostNetwork: false,
+				PortMappings: []*PortMapping{
+					{
+						HostPort:      4567,
+						ContainerPort: 80,
+						Protocol:      v1.ProtocolTCP,
+					},
+					{
+						HostPort:      5678,
+						ContainerPort: 81,
+						Protocol:      v1.ProtocolUDP,
+					},
+				},
+			},
+			[]*ruleMatch{
+				{
+					-1,
+					"KUBE-HOSTPORTS",
+					"-m comment --comment \"test-pod_default hostport 4567\" -m tcp -p tcp --dport 4567",
+				},
+				{
+					4567,
+					"",
+					"-m comment --comment \"test-pod_default hostport 4567\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ",
+				},
+				{
+					4567,
+					"",
+					"-m comment --comment \"test-pod_default hostport 4567\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80",
+				},
+				{
+					-1,
+					"KUBE-HOSTPORTS",
+					"-m comment --comment \"test-pod_default hostport 5678\" -m udp -p udp --dport 5678",
+				},
+				{
+					5678,
+					"",
+					"-m comment --comment \"test-pod_default hostport 5678\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ",
+				},
+				{
+					5678,
+					"",
+					"-m comment --comment \"test-pod_default hostport 5678\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81",
+				},
+			},
+		},
+		// Already running pod
+		{
+			&PodPortMapping{
+				Name:        "another-test-pod",
+				Namespace:   v1.NamespaceDefault,
+				IP:          net.ParseIP("10.1.1.5"),
+				HostNetwork: false,
+				PortMappings: []*PortMapping{
+					{
+						HostPort:      123,
+						ContainerPort: 654,
+						Protocol:      v1.ProtocolTCP,
+					},
+				},
+			},
+			[]*ruleMatch{
+				{
+					-1,
+					"KUBE-HOSTPORTS",
+					"-m comment --comment \"another-test-pod_default hostport 123\" -m tcp -p tcp --dport 123",
+				},
+				{
+					123,
+					"",
+					"-m comment --comment \"another-test-pod_default hostport 123\" -s 10.1.1.5/32 -j KUBE-MARK-MASQ",
+				},
+				{
+					123,
+					"",
+					"-m comment --comment \"another-test-pod_default hostport 123\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.5:654",
+				},
+			},
+		},
+	}
+
+	activePodPortMapping := make([]*PodPortMapping, 0)
+
+	// Fill in any match rules missing chain names
+	for _, test := range tests {
+		for _, match := range test.matches {
+			if match.hostport >= 0 {
+				found := false
+				for _, pm := range test.mapping.PortMappings {
+					if int(pm.HostPort) == match.hostport {
+						match.chain = string(hostportChainName(pm, getPodFullName(test.mapping)))
+						found = true
+						break
+					}
+				}
+				if !found {
+					t.Fatalf("Failed to find ContainerPort for match %d/'%s'", match.hostport, match.match)
+				}
+			}
+		}
+		activePodPortMapping = append(activePodPortMapping, test.mapping)
+
+	}
+
+	// Already running pod's host port
+	hp := hostport{
+		tests[1].mapping.PortMappings[0].HostPort,
+		strings.ToLower(string(tests[1].mapping.PortMappings[0].Protocol)),
+	}
+	h.hostPortMap[hp] = &fakeSocket{
+		tests[1].mapping.PortMappings[0].HostPort,
+		strings.ToLower(string(tests[1].mapping.PortMappings[0].Protocol)),
+		false,
+	}
+
+	err := h.OpenPodHostportsAndSync(tests[0].mapping, "br0", activePodPortMapping)
+	if err != nil {
+		t.Fatalf("Failed to OpenPodHostportsAndSync: %v", err)
+	}
+
+	// Generic rules
+	genericRules := []*ruleMatch{
+		{-1, "POSTROUTING", "-m comment --comment \"SNAT for localhost access to hostports\" -o br0 -s 127.0.0.0/8 -j MASQUERADE"},
+		{-1, "PREROUTING", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+		{-1, "OUTPUT", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+	}
+
+	for _, rule := range genericRules {
+		_, chain, err := fakeIPTables.getChain(utiliptables.TableNAT, utiliptables.Chain(rule.chain))
+		if err != nil {
+			t.Fatalf("Expected NAT chain %s did not exist", rule.chain)
+		}
+		if !matchRule(chain, rule.match) {
+			t.Fatalf("Expected %s chain rule match '%s' not found", rule.chain, rule.match)
+		}
+	}
+
+	// Pod rules
+	for _, test := range tests {
+		for _, match := range test.matches {
+			// Ensure chain exists
+			_, chain, err := fakeIPTables.getChain(utiliptables.TableNAT, utiliptables.Chain(match.chain))
+			if err != nil {
+				t.Fatalf("Expected NAT chain %s did not exist", match.chain)
+			}
+			if !matchRule(chain, match.match) {
+				t.Fatalf("Expected NAT chain %s rule containing '%s' not found", match.chain, match.match)
+			}
+		}
+	}
+
+	// Socket
+	hostPortMap := map[hostport]closeable{
+		{123, "tcp"}:  &fakeSocket{123, "tcp", false},
+		{4567, "tcp"}: &fakeSocket{4567, "tcp", false},
+		{5678, "udp"}: &fakeSocket{5678, "udp", false},
+	}
+	if !reflect.DeepEqual(hostPortMap, h.hostPortMap) {
+		t.Fatalf("Mismatch in expected hostPortMap. Expected '%v', got '%v'", hostPortMap, h.hostPortMap)
+	}
+}
+
+func matchRule(chain *fakeChain, match string) bool {
+	for _, rule := range chain.rules {
+		if strings.Contains(rule, match) {
+			return true
+		}
+	}
+	return false
+}
+
+func TestHostportChainName(t *testing.T) {
+	m := make(map[string]int)
+	chain := hostportChainName(&PortMapping{HostPort: 57119, Protocol: "TCP", ContainerPort: 57119}, "testrdma-2")
+	m[string(chain)] = 1
+	chain = hostportChainName(&PortMapping{HostPort: 55429, Protocol: "TCP", ContainerPort: 55429}, "testrdma-2")
+	m[string(chain)] = 1
+	chain = hostportChainName(&PortMapping{HostPort: 56833, Protocol: "TCP", ContainerPort: 56833}, "testrdma-2")
+	m[string(chain)] = 1
+	if len(m) != 3 {
+		t.Fatal(m)
+	}
+}
+
+func TestHostPortSyncerRemoveLegacyRules(t *testing.T) {
+	iptables := NewFakeIPTables()
+	legacyRules := [][]string{
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j KUBE-HP-5N7UH5JAXCVP5UJR"},
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp --dport 8081 -j KUBE-HP-7THKRFSEH4GIIXK7"},
+		{"-A", "KUBE-HOSTPORTS", "-m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp --dport 8080 -j KUBE-HP-4YVONL46AKYWSKS3"},
+		{"-A", "OUTPUT", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+		{"-A", "PREROUTING", "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"},
+		{"-A", "POSTROUTING", "-m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE"},
+		{"-A", "KUBE-HP-4YVONL46AKYWSKS3", "-m comment --comment \"pod1_ns1 hostport 8080\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-4YVONL46AKYWSKS3", "-m comment --comment \"pod1_ns1 hostport 8080\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.2:80"},
+		{"-A", "KUBE-HP-7THKRFSEH4GIIXK7", "-m comment --comment \"pod1_ns1 hostport 8081\" -s 10.1.1.2/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-7THKRFSEH4GIIXK7", "-m comment --comment \"pod1_ns1 hostport 8081\" -m udp -p udp -j DNAT --to-destination 10.1.1.2:81"},
+		{"-A", "KUBE-HP-5N7UH5JAXCVP5UJR", "-m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ"},
+		{"-A", "KUBE-HP-5N7UH5JAXCVP5UJR", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443"},
+	}
+	for _, rule := range legacyRules {
+		_, err := iptables.EnsureChain(utiliptables.TableNAT, utiliptables.Chain(rule[1]))
+		assert.NoError(t, err)
+		_, err = iptables.ensureRule(utiliptables.RulePosition(rule[0]), utiliptables.TableNAT, utiliptables.Chain(rule[1]), rule[2])
+		assert.NoError(t, err)
+	}
+	portOpener := NewFakeSocketManager()
+	h := &hostportSyncer{
+		hostPortMap: make(map[hostport]closeable),
+		iptables:    iptables,
+		portOpener:  portOpener.openFakeSocket,
+	}
+	// check preserve pod3's rules and remove pod1's rules
+	pod3PortMapping := &PodPortMapping{
+		Name:        "pod3",
+		Namespace:   "ns1",
+		IP:          net.ParseIP("10.1.1.4"),
+		HostNetwork: false,
+		PortMappings: []*PortMapping{
+			{
+				HostPort:      8443,
+				ContainerPort: 443,
+				Protocol:      v1.ProtocolTCP,
+			},
+		},
+	}
+	h.SyncHostports("cbr0", []*PodPortMapping{pod3PortMapping})
+
+	newChainName := string(hostportChainName(pod3PortMapping.PortMappings[0], getPodFullName(pod3PortMapping)))
+	expectRules := [][]string{
+		{"KUBE-HOSTPORTS", "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp --dport 8443 -j " + newChainName},
+		{newChainName, "-m comment --comment \"pod3_ns1 hostport 8443\" -s 10.1.1.4/32 -j KUBE-MARK-MASQ"},
+		{newChainName, "-m comment --comment \"pod3_ns1 hostport 8443\" -m tcp -p tcp -j DNAT --to-destination 10.1.1.4:443"},
+	}
+
+	natTable, ok := iptables.tables[string(utiliptables.TableNAT)]
+	assert.True(t, ok)
+	// check pod1's rules in KUBE-HOSTPORTS chain should be cleaned up
+	hostportChain, ok := natTable.chains["KUBE-HOSTPORTS"]
+	assert.True(t, ok, string(hostportChain.name))
+	assert.Equal(t, 1, len(hostportChain.rules), "%v", hostportChain.rules)
+
+	// check pod3's rules left
+	assert.Equal(t, expectRules[0][1], hostportChain.rules[0])
+	chain, ok := natTable.chains[newChainName]
+	assert.True(t, ok)
+	assert.Equal(t, 2, len(chain.rules))
+	assert.Equal(t, expectRules[1][1], chain.rules[0])
+	assert.Equal(t, expectRules[2][1], chain.rules[1])
+
+	// check legacy KUBE-HP-* chains should be deleted
+	for _, name := range []string{"KUBE-HP-4YVONL46AKYWSKS3", "KUBE-HP-7THKRFSEH4GIIXK7", "KUBE-HP-5N7UH5JAXCVP5UJR"} {
+		_, ok := natTable.chains[name]
+		assert.False(t, ok)
+	}
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/hostport_test.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostport
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/api/core/v1"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+)
+
+type fakeSocket struct {
+	port     int32
+	protocol string
+	closed   bool
+}
+
+func (f *fakeSocket) Close() error {
+	if f.closed {
+		return fmt.Errorf("Socket %q.%s already closed!", f.port, f.protocol)
+	}
+	f.closed = true
+	return nil
+}
+
+func NewFakeSocketManager() *fakeSocketManager {
+	return &fakeSocketManager{mem: make(map[hostport]*fakeSocket)}
+}
+
+type fakeSocketManager struct {
+	mem map[hostport]*fakeSocket
+}
+
+func (f *fakeSocketManager) openFakeSocket(hp *hostport) (closeable, error) {
+	if socket, ok := f.mem[*hp]; ok && !socket.closed {
+		return nil, fmt.Errorf("hostport is occupied")
+	}
+	fs := &fakeSocket{hp.port, hp.protocol, false}
+	f.mem[*hp] = fs
+	return fs, nil
+}
+
+func TestOpenHostports(t *testing.T) {
+	opener := NewFakeSocketManager()
+	testCases := []struct {
+		podPortMapping *PodPortMapping
+		expectError    bool
+	}{
+		{
+			&PodPortMapping{
+				Namespace: "ns1",
+				Name:      "n0",
+			},
+			false,
+		},
+		{
+			&PodPortMapping{
+				Namespace: "ns1",
+				Name:      "n1",
+				PortMappings: []*PortMapping{
+					{HostPort: 80, Protocol: v1.Protocol("TCP")},
+					{HostPort: 8080, Protocol: v1.Protocol("TCP")},
+					{HostPort: 443, Protocol: v1.Protocol("TCP")},
+				},
+			},
+			false,
+		},
+		{
+			&PodPortMapping{
+				Namespace: "ns1",
+				Name:      "n2",
+				PortMappings: []*PortMapping{
+					{HostPort: 80, Protocol: v1.Protocol("TCP")},
+				},
+			},
+			true,
+		},
+		{
+			&PodPortMapping{
+				Namespace: "ns1",
+				Name:      "n3",
+				PortMappings: []*PortMapping{
+					{HostPort: 8081, Protocol: v1.Protocol("TCP")},
+					{HostPort: 8080, Protocol: v1.Protocol("TCP")},
+				},
+			},
+			true,
+		},
+		{
+			&PodPortMapping{
+				Namespace: "ns1",
+				Name:      "n3",
+				PortMappings: []*PortMapping{
+					{HostPort: 8081, Protocol: v1.Protocol("TCP")},
+				},
+			},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		mapping, err := openHostports(opener.openFakeSocket, tc.podPortMapping)
+		if tc.expectError {
+			assert.Error(t, err)
+			continue
+		}
+		assert.NoError(t, err)
+		assert.EqualValues(t, len(mapping), len(tc.podPortMapping.PortMappings))
+	}
+}
+
+func TestEnsureKubeHostportChains(t *testing.T) {
+	interfaceName := "cbr0"
+	builtinChains := []string{"PREROUTING", "OUTPUT"}
+	jumpRule := "-m comment --comment \"kube hostport portals\" -m addrtype --dst-type LOCAL -j KUBE-HOSTPORTS"
+	masqRule := "-m comment --comment \"SNAT for localhost access to hostports\" -o cbr0 -s 127.0.0.0/8 -j MASQUERADE"
+
+	fakeIPTables := NewFakeIPTables()
+	assert.NoError(t, ensureKubeHostportChains(fakeIPTables, interfaceName))
+
+	_, _, err := fakeIPTables.getChain(utiliptables.TableNAT, utiliptables.Chain("KUBE-HOSTPORTS"))
+	assert.NoError(t, err)
+
+	_, chain, err := fakeIPTables.getChain(utiliptables.TableNAT, utiliptables.ChainPostrouting)
+	assert.NoError(t, err)
+	assert.EqualValues(t, len(chain.rules), 1)
+	assert.Contains(t, chain.rules[0], masqRule)
+
+	for _, chainName := range builtinChains {
+		_, chain, err := fakeIPTables.getChain(utiliptables.TableNAT, utiliptables.Chain(chainName))
+		assert.NoError(t, err)
+		assert.EqualValues(t, len(chain.rules), 1)
+		assert.Contains(t, chain.rules[0], jumpRule)
+	}
+
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/testing/BUILD

@@ -0,0 +1,26 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["fake.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/hostport/testing",
+    deps = ["//pkg/kubelet/network/hostport:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/hostport/testing/fake.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
+)
+
+type fakeSyncer struct{}
+
+func NewFakeHostportSyncer() hostport.HostportSyncer {
+	return &fakeSyncer{}
+}
+
+func (h *fakeSyncer) OpenPodHostportsAndSync(newPortMapping *hostport.PodPortMapping, natInterfaceName string, activePortMapping []*hostport.PodPortMapping) error {
+	return h.SyncHostports(natInterfaceName, activePortMapping)
+}
+
+func (h *fakeSyncer) SyncHostports(natInterfaceName string, activePortMapping []*hostport.PodPortMapping) error {
+	for _, r := range activePortMapping {
+		if r.IP.To4() == nil {
+			return fmt.Errorf("Invalid or missing pod %s/%s IP", r.Namespace, r.Name)
+		}
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/kubenet/BUILD

@@ -0,0 +1,90 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "kubenet.go",
+        "kubenet_unsupported.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "kubenet_linux.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/kubenet",
+    deps = [
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/network:go_default_library",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//pkg/kubelet/network/hostport:go_default_library",
+            "//pkg/util/bandwidth:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//pkg/util/ebtables:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//pkg/util/sysctl:go_default_library",
+            "//vendor/github.com/containernetworking/cni/libcni:go_default_library",
+            "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
+            "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
+            "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/github.com/vishvananda/netlink:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/api/core/v1:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "kubenet_linux_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/kubenet",
+    library = ":go_default_library",
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux_amd64": [
+            "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+            "//pkg/kubelet/container:go_default_library",
+            "//pkg/kubelet/network:go_default_library",
+            "//pkg/kubelet/network/cni/testing:go_default_library",
+            "//pkg/kubelet/network/hostport/testing:go_default_library",
+            "//pkg/kubelet/network/testing:go_default_library",
+            "//pkg/util/bandwidth:go_default_library",
+            "//pkg/util/iptables/testing:go_default_library",
+            "//pkg/util/sysctl/testing:go_default_library",
+            "//vendor/github.com/stretchr/testify/assert:go_default_library",
+            "//vendor/github.com/stretchr/testify/mock:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/kubenet/kubenet.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubenet
+
+const (
+	KubenetPluginName = "kubenet"
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/kubenet/kubenet_linux.go

@@ -0,0 +1,867 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubenet
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/containernetworking/cni/libcni"
+	cnitypes "github.com/containernetworking/cni/pkg/types"
+	cnitypes020 "github.com/containernetworking/cni/pkg/types/020"
+	"github.com/golang/glog"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+	"k8s.io/api/core/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	utilsets "k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
+	"k8s.io/kubernetes/pkg/util/bandwidth"
+	utildbus "k8s.io/kubernetes/pkg/util/dbus"
+	utilebtables "k8s.io/kubernetes/pkg/util/ebtables"
+	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
+	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
+	utilexec "k8s.io/utils/exec"
+)
+
+const (
+	BridgeName    = "cbr0"
+	DefaultCNIDir = "/opt/cni/bin"
+
+	sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
+
+	// fallbackMTU is used if an MTU is not specified, and we cannot determine the MTU
+	fallbackMTU = 1460
+
+	// ebtables Chain to store dedup rules
+	dedupChain = utilebtables.Chain("KUBE-DEDUP")
+
+	// defaultIPAMDir is the default location for the checkpoint files stored by host-local ipam
+	// https://github.com/containernetworking/cni/tree/master/plugins/ipam/host-local#backends
+	defaultIPAMDir = "/var/lib/cni/networks"
+)
+
+// CNI plugins required by kubenet in /opt/cni/bin or vendor directory
+var requiredCNIPlugins = [...]string{"bridge", "host-local", "loopback"}
+
+type kubenetNetworkPlugin struct {
+	network.NoopNetworkPlugin
+
+	host            network.Host
+	netConfig       *libcni.NetworkConfig
+	loConfig        *libcni.NetworkConfig
+	cniConfig       libcni.CNI
+	bandwidthShaper bandwidth.BandwidthShaper
+	mu              sync.Mutex //Mutex for protecting podIPs map, netConfig, and shaper initialization
+	podIPs          map[kubecontainer.ContainerID]string
+	mtu             int
+	execer          utilexec.Interface
+	nsenterPath     string
+	hairpinMode     kubeletconfig.HairpinMode
+	// kubenet can use either hostportSyncer and hostportManager to implement hostports
+	// Currently, if network host supports legacy features, hostportSyncer will be used,
+	// otherwise, hostportManager will be used.
+	hostportSyncer  hostport.HostportSyncer
+	hostportManager hostport.HostPortManager
+	iptables        utiliptables.Interface
+	sysctl          utilsysctl.Interface
+	ebtables        utilebtables.Interface
+	// vendorDir is passed by kubelet cni-bin-dir parameter.
+	// kubenet will search for cni binaries in DefaultCNIDir first, then continue to vendorDir.
+	vendorDir         string
+	nonMasqueradeCIDR string
+	podCidr           string
+	gateway           net.IP
+}
+
+func NewPlugin(networkPluginDir string) network.NetworkPlugin {
+	protocol := utiliptables.ProtocolIpv4
+	execer := utilexec.New()
+	dbus := utildbus.New()
+	sysctl := utilsysctl.New()
+	iptInterface := utiliptables.New(execer, dbus, protocol)
+	return &kubenetNetworkPlugin{
+		podIPs:            make(map[kubecontainer.ContainerID]string),
+		execer:            utilexec.New(),
+		iptables:          iptInterface,
+		sysctl:            sysctl,
+		vendorDir:         networkPluginDir,
+		hostportSyncer:    hostport.NewHostportSyncer(iptInterface),
+		hostportManager:   hostport.NewHostportManager(iptInterface),
+		nonMasqueradeCIDR: "10.0.0.0/8",
+	}
+}
+
+func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	plugin.host = host
+	plugin.hairpinMode = hairpinMode
+	plugin.nonMasqueradeCIDR = nonMasqueradeCIDR
+	plugin.cniConfig = &libcni.CNIConfig{
+		Path: []string{DefaultCNIDir, plugin.vendorDir},
+	}
+
+	if mtu == network.UseDefaultMTU {
+		if link, err := findMinMTU(); err == nil {
+			plugin.mtu = link.MTU
+			glog.V(5).Infof("Using interface %s MTU %d as bridge MTU", link.Name, link.MTU)
+		} else {
+			plugin.mtu = fallbackMTU
+			glog.Warningf("Failed to find default bridge MTU, using %d: %v", fallbackMTU, err)
+		}
+	} else {
+		plugin.mtu = mtu
+	}
+
+	// Since this plugin uses a Linux bridge, set bridge-nf-call-iptables=1
+	// is necessary to ensure kube-proxy functions correctly.
+	//
+	// This will return an error on older kernel version (< 3.18) as the module
+	// was built-in, we simply ignore the error here. A better thing to do is
+	// to check the kernel version in the future.
+	plugin.execer.Command("modprobe", "br-netfilter").CombinedOutput()
+	err := plugin.sysctl.SetSysctl(sysctlBridgeCallIPTables, 1)
+	if err != nil {
+		glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err)
+	}
+
+	plugin.loConfig, err = libcni.ConfFromBytes([]byte(`{
+  "cniVersion": "0.1.0",
+  "name": "kubenet-loopback",
+  "type": "loopback"
+}`))
+	if err != nil {
+		return fmt.Errorf("Failed to generate loopback config: %v", err)
+	}
+
+	plugin.nsenterPath, err = plugin.execer.LookPath("nsenter")
+	if err != nil {
+		return fmt.Errorf("Failed to find nsenter binary: %v", err)
+	}
+
+	// Need to SNAT outbound traffic from cluster
+	if err = plugin.ensureMasqRule(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// TODO: move thic logic into cni bridge plugin and remove this from kubenet
+func (plugin *kubenetNetworkPlugin) ensureMasqRule() error {
+	if plugin.nonMasqueradeCIDR != "0.0.0.0/0" {
+		if _, err := plugin.iptables.EnsureRule(utiliptables.Append, utiliptables.TableNAT, utiliptables.ChainPostrouting,
+			"-m", "comment", "--comment", "kubenet: SNAT for outbound traffic from cluster",
+			"-m", "addrtype", "!", "--dst-type", "LOCAL",
+			"!", "-d", plugin.nonMasqueradeCIDR,
+			"-j", "MASQUERADE"); err != nil {
+			return fmt.Errorf("Failed to ensure that %s chain %s jumps to MASQUERADE: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, err)
+		}
+	}
+	return nil
+}
+
+func findMinMTU() (*net.Interface, error) {
+	intfs, err := net.Interfaces()
+	if err != nil {
+		return nil, err
+	}
+
+	mtu := 999999
+	defIntfIndex := -1
+	for i, intf := range intfs {
+		if ((intf.Flags & net.FlagUp) != 0) && (intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0) {
+			if intf.MTU < mtu {
+				mtu = intf.MTU
+				defIntfIndex = i
+			}
+		}
+	}
+
+	if mtu >= 999999 || mtu < 576 || defIntfIndex < 0 {
+		return nil, fmt.Errorf("no suitable interface: %v", BridgeName)
+	}
+
+	return &intfs[defIntfIndex], nil
+}
+
+const NET_CONFIG_TEMPLATE = `{
+  "cniVersion": "0.1.0",
+  "name": "kubenet",
+  "type": "bridge",
+  "bridge": "%s",
+  "mtu": %d,
+  "addIf": "%s",
+  "isGateway": true,
+  "ipMasq": false,
+  "hairpinMode": %t,
+  "ipam": {
+    "type": "host-local",
+    "subnet": "%s",
+    "gateway": "%s",
+    "routes": [
+      { "dst": "0.0.0.0/0" }
+    ]
+  }
+}`
+
+func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interface{}) {
+	if name != network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE {
+		return
+	}
+
+	plugin.mu.Lock()
+	defer plugin.mu.Unlock()
+
+	podCIDR, ok := details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR].(string)
+	if !ok {
+		glog.Warningf("%s event didn't contain pod CIDR", network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE)
+		return
+	}
+
+	if plugin.netConfig != nil {
+		glog.Warningf("Ignoring subsequent pod CIDR update to %s", podCIDR)
+		return
+	}
+
+	glog.V(5).Infof("PodCIDR is set to %q", podCIDR)
+	_, cidr, err := net.ParseCIDR(podCIDR)
+	if err == nil {
+		setHairpin := plugin.hairpinMode == kubeletconfig.HairpinVeth
+		// Set bridge address to first address in IPNet
+		cidr.IP[len(cidr.IP)-1] += 1
+
+		json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, network.DefaultInterfaceName, setHairpin, podCIDR, cidr.IP.String())
+		glog.V(2).Infof("CNI network config set to %v", json)
+		plugin.netConfig, err = libcni.ConfFromBytes([]byte(json))
+		if err == nil {
+			glog.V(5).Infof("CNI network config:\n%s", json)
+
+			// Ensure cbr0 has no conflicting addresses; CNI's 'bridge'
+			// plugin will bail out if the bridge has an unexpected one
+			plugin.clearBridgeAddressesExcept(cidr)
+		}
+		plugin.podCidr = podCIDR
+		plugin.gateway = cidr.IP
+	}
+
+	if err != nil {
+		glog.Warningf("Failed to generate CNI network config: %v", err)
+	}
+}
+
+func (plugin *kubenetNetworkPlugin) clearBridgeAddressesExcept(keep *net.IPNet) {
+	bridge, err := netlink.LinkByName(BridgeName)
+	if err != nil {
+		return
+	}
+
+	addrs, err := netlink.AddrList(bridge, unix.AF_INET)
+	if err != nil {
+		return
+	}
+
+	for _, addr := range addrs {
+		if !utilnet.IPNetEqual(addr.IPNet, keep) {
+			glog.V(2).Infof("Removing old address %s from %s", addr.IPNet.String(), BridgeName)
+			netlink.AddrDel(bridge, &addr)
+		}
+	}
+}
+
+func (plugin *kubenetNetworkPlugin) Name() string {
+	return KubenetPluginName
+}
+
+func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int {
+	return utilsets.NewInt()
+}
+
+// setup sets up networking through CNI using the given ns/name and sandbox ID.
+// TODO: Don't pass the pod to this method, it only needs it for bandwidth
+// shaping and hostport management.
+func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, pod *v1.Pod, annotations map[string]string) error {
+	// Disable DAD so we skip the kernel delay on bringing up new interfaces.
+	if err := plugin.disableContainerDAD(id); err != nil {
+		glog.V(3).Infof("Failed to disable DAD in container: %v", err)
+	}
+
+	// Bring up container loopback interface
+	if _, err := plugin.addContainerToNetwork(plugin.loConfig, "lo", namespace, name, id); err != nil {
+		return err
+	}
+
+	// Hook container up with our bridge
+	resT, err := plugin.addContainerToNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id)
+	if err != nil {
+		return err
+	}
+	// Coerce the CNI result version
+	res, err := cnitypes020.GetResult(resT)
+	if err != nil {
+		return fmt.Errorf("unable to understand network config: %v", err)
+	}
+	if res.IP4 == nil {
+		return fmt.Errorf("CNI plugin reported no IPv4 address for container %v.", id)
+	}
+	ip4 := res.IP4.IP.IP.To4()
+	if ip4 == nil {
+		return fmt.Errorf("CNI plugin reported an invalid IPv4 address for container %v: %+v.", id, res.IP4)
+	}
+
+	// Put the container bridge into promiscuous mode to force it to accept hairpin packets.
+	// TODO: Remove this once the kernel bug (#20096) is fixed.
+	// TODO: check and set promiscuous mode with netlink once vishvananda/netlink supports it
+	if plugin.hairpinMode == kubeletconfig.PromiscuousBridge {
+		output, err := plugin.execer.Command("ip", "link", "show", "dev", BridgeName).CombinedOutput()
+		if err != nil || strings.Index(string(output), "PROMISC") < 0 {
+			_, err := plugin.execer.Command("ip", "link", "set", BridgeName, "promisc", "on").CombinedOutput()
+			if err != nil {
+				return fmt.Errorf("Error setting promiscuous mode on %s: %v", BridgeName, err)
+			}
+		}
+
+		link, err := netlink.LinkByName(BridgeName)
+		if err != nil {
+			return fmt.Errorf("failed to lookup %q: %v", BridgeName, err)
+		}
+
+		// configure the ebtables rules to eliminate duplicate packets by best effort
+		plugin.syncEbtablesDedupRules(link.Attrs().HardwareAddr)
+	}
+
+	plugin.podIPs[id] = ip4.String()
+
+	// The first SetUpPod call creates the bridge; get a shaper for the sake of initialization
+	// TODO: replace with CNI traffic shaper plugin
+	shaper := plugin.shaper()
+	ingress, egress, err := bandwidth.ExtractPodBandwidthResources(annotations)
+	if err != nil {
+		return fmt.Errorf("Error reading pod bandwidth annotations: %v", err)
+	}
+	if egress != nil || ingress != nil {
+		if err := shaper.ReconcileCIDR(fmt.Sprintf("%s/32", ip4.String()), egress, ingress); err != nil {
+			return fmt.Errorf("Failed to add pod to shaper: %v", err)
+		}
+	}
+
+	// The host can choose to not support "legacy" features. The remote
+	// shim doesn't support it (#35457), but the kubelet does.
+	if plugin.host.SupportsLegacyFeatures() {
+		// Open any hostport the pod's containers want
+		activePodPortMappings, err := plugin.getPodPortMappings()
+		if err != nil {
+			return err
+		}
+
+		newPodPortMapping := hostport.ConstructPodPortMapping(pod, ip4)
+		if err := plugin.hostportSyncer.OpenPodHostportsAndSync(newPodPortMapping, BridgeName, activePodPortMappings); err != nil {
+			return err
+		}
+	} else {
+		// TODO: replace with CNI port-forwarding plugin
+		portMappings, err := plugin.host.GetPodPortMappings(id.ID)
+		if err != nil {
+			return err
+		}
+		if portMappings != nil && len(portMappings) > 0 {
+			if err := plugin.hostportManager.Add(id.ID, &hostport.PodPortMapping{
+				Namespace:    namespace,
+				Name:         name,
+				PortMappings: portMappings,
+				IP:           ip4,
+				HostNetwork:  false,
+			}, BridgeName); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	plugin.mu.Lock()
+	defer plugin.mu.Unlock()
+
+	start := time.Now()
+	defer func() {
+		glog.V(4).Infof("SetUpPod took %v for %s/%s", time.Since(start), namespace, name)
+	}()
+
+	// TODO: Entire pod object only required for bw shaping and hostport.
+	pod, ok := plugin.host.GetPodByName(namespace, name)
+	if !ok {
+		return fmt.Errorf("pod %q cannot be found", name)
+	}
+
+	if err := plugin.Status(); err != nil {
+		return fmt.Errorf("Kubenet cannot SetUpPod: %v", err)
+	}
+
+	if err := plugin.setup(namespace, name, id, pod, annotations); err != nil {
+		// Make sure everything gets cleaned up on errors
+		podIP, _ := plugin.podIPs[id]
+		if err := plugin.teardown(namespace, name, id, podIP); err != nil {
+			// Not a hard error or warning
+			glog.V(4).Infof("Failed to clean up %s/%s after SetUpPod failure: %v", namespace, name, err)
+		}
+
+		// TODO(#34278): Figure out if we need IP GC through the cri.
+		// The cri should always send us teardown events for stale sandboxes,
+		// this obviates the need for GC in the common case, for kubenet.
+		if plugin.host.SupportsLegacyFeatures() {
+
+			// TODO: Remove this hack once we've figured out how to retrieve the netns
+			// of an exited container. Currently, restarting docker will leak a bunch of
+			// ips. This will exhaust available ip space unless we cleanup old ips. At the
+			// same time we don't want to try GC'ing them periodically as that could lead
+			// to a performance regression in starting pods. So on each setup failure, try
+			// GC on the assumption that the kubelet is going to retry pod creation, and
+			// when it does, there will be ips.
+			plugin.ipamGarbageCollection()
+		}
+		return err
+	}
+
+	// Need to SNAT outbound traffic from cluster
+	if err := plugin.ensureMasqRule(); err != nil {
+		glog.Errorf("Failed to ensure MASQ rule: %v", err)
+	}
+
+	return nil
+}
+
+// Tears down as much of a pod's network as it can even if errors occur.  Returns
+// an aggregate error composed of all errors encountered during the teardown.
+func (plugin *kubenetNetworkPlugin) teardown(namespace string, name string, id kubecontainer.ContainerID, podIP string) error {
+	errList := []error{}
+
+	if podIP != "" {
+		glog.V(5).Infof("Removing pod IP %s from shaper", podIP)
+		// shaper wants /32
+		if err := plugin.shaper().Reset(fmt.Sprintf("%s/32", podIP)); err != nil {
+			// Possible bandwidth shaping wasn't enabled for this pod anyways
+			glog.V(4).Infof("Failed to remove pod IP %s from shaper: %v", podIP, err)
+		}
+
+		delete(plugin.podIPs, id)
+	}
+
+	if err := plugin.delContainerFromNetwork(plugin.netConfig, network.DefaultInterfaceName, namespace, name, id); err != nil {
+		// This is to prevent returning error when TearDownPod is called twice on the same pod. This helps to reduce event pollution.
+		if podIP != "" {
+			glog.Warningf("Failed to delete container from kubenet: %v", err)
+		} else {
+			errList = append(errList, err)
+		}
+	}
+
+	// The host can choose to not support "legacy" features. The remote
+	// shim doesn't support it (#35457), but the kubelet does.
+	if plugin.host.SupportsLegacyFeatures() {
+		activePodPortMapping, err := plugin.getPodPortMappings()
+		if err == nil {
+			err = plugin.hostportSyncer.SyncHostports(BridgeName, activePodPortMapping)
+		}
+		if err != nil {
+			errList = append(errList, err)
+		}
+	} else {
+		portMappings, err := plugin.host.GetPodPortMappings(id.ID)
+		if err != nil {
+			errList = append(errList, err)
+		} else if portMappings != nil && len(portMappings) > 0 {
+			if err = plugin.hostportManager.Remove(id.ID, &hostport.PodPortMapping{
+				Namespace:    namespace,
+				Name:         name,
+				PortMappings: portMappings,
+				HostNetwork:  false,
+			}); err != nil {
+				errList = append(errList, err)
+			}
+		}
+	}
+	return utilerrors.NewAggregate(errList)
+}
+
+func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.ContainerID) error {
+	plugin.mu.Lock()
+	defer plugin.mu.Unlock()
+
+	start := time.Now()
+	defer func() {
+		glog.V(4).Infof("TearDownPod took %v for %s/%s", time.Since(start), namespace, name)
+	}()
+
+	if plugin.netConfig == nil {
+		return fmt.Errorf("Kubenet needs a PodCIDR to tear down pods")
+	}
+
+	// no cached IP is Ok during teardown
+	podIP, _ := plugin.podIPs[id]
+	if err := plugin.teardown(namespace, name, id, podIP); err != nil {
+		return err
+	}
+
+	// Need to SNAT outbound traffic from cluster
+	if err := plugin.ensureMasqRule(); err != nil {
+		glog.Errorf("Failed to ensure MASQ rule: %v", err)
+	}
+
+	return nil
+}
+
+// TODO: Use the addToNetwork function to obtain the IP of the Pod. That will assume idempotent ADD call to the plugin.
+// Also fix the runtime's call to Status function to be done only in the case that the IP is lost, no need to do periodic calls
+func (plugin *kubenetNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	plugin.mu.Lock()
+	defer plugin.mu.Unlock()
+	// Assuming the ip of pod does not change. Try to retrieve ip from kubenet map first.
+	if podIP, ok := plugin.podIPs[id]; ok {
+		return &network.PodNetworkStatus{IP: net.ParseIP(podIP)}, nil
+	}
+
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return nil, fmt.Errorf("Kubenet failed to retrieve network namespace path: %v", err)
+	}
+	if netnsPath == "" {
+		return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id)
+	}
+	ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
+	if err != nil {
+		return nil, err
+	}
+
+	plugin.podIPs[id] = ip.String()
+	return &network.PodNetworkStatus{IP: ip}, nil
+}
+
+func (plugin *kubenetNetworkPlugin) Status() error {
+	// Can't set up pods if we don't have a PodCIDR yet
+	if plugin.netConfig == nil {
+		return fmt.Errorf("Kubenet does not have netConfig. This is most likely due to lack of PodCIDR")
+	}
+
+	if !plugin.checkCNIPlugin() {
+		return fmt.Errorf("could not locate kubenet required CNI plugins %v at %q or %q", requiredCNIPlugins, DefaultCNIDir, plugin.vendorDir)
+	}
+	return nil
+}
+
+// checkCNIPlugin returns if all kubenet required cni plugins can be found at /opt/cni/bin or user specifed NetworkPluginDir.
+func (plugin *kubenetNetworkPlugin) checkCNIPlugin() bool {
+	if plugin.checkCNIPluginInDir(DefaultCNIDir) || plugin.checkCNIPluginInDir(plugin.vendorDir) {
+		return true
+	}
+	return false
+}
+
+// checkCNIPluginInDir returns if all required cni plugins are placed in dir
+func (plugin *kubenetNetworkPlugin) checkCNIPluginInDir(dir string) bool {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return false
+	}
+	for _, cniPlugin := range requiredCNIPlugins {
+		found := false
+		for _, file := range files {
+			if strings.TrimSpace(file.Name()) == cniPlugin {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+	return true
+}
+
+// getNonExitedPods returns a list of pods that have at least one running container.
+func (plugin *kubenetNetworkPlugin) getNonExitedPods() ([]*kubecontainer.Pod, error) {
+	ret := []*kubecontainer.Pod{}
+	pods, err := plugin.host.GetRuntime().GetPods(true)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to retrieve pods from runtime: %v", err)
+	}
+	for _, p := range pods {
+		if podIsExited(p) {
+			continue
+		}
+		ret = append(ret, p)
+	}
+	return ret, nil
+}
+
+func (plugin *kubenetNetworkPlugin) getPodPortMappings() ([]*hostport.PodPortMapping, error) {
+	pods, err := plugin.getNonExitedPods()
+	if err != nil {
+		return nil, err
+	}
+	activePodPortMappings := make([]*hostport.PodPortMapping, 0)
+	for _, p := range pods {
+		containerID, err := plugin.host.GetRuntime().GetPodContainerID(p)
+		if err != nil {
+			continue
+		}
+		ipString, ok := plugin.podIPs[containerID]
+		if !ok {
+			continue
+		}
+		podIP := net.ParseIP(ipString)
+		if podIP == nil {
+			continue
+		}
+		if pod, ok := plugin.host.GetPodByName(p.Namespace, p.Name); ok {
+			activePodPortMappings = append(activePodPortMappings, hostport.ConstructPodPortMapping(pod, podIP))
+		}
+	}
+	return activePodPortMappings, nil
+}
+
+// ipamGarbageCollection will release unused IP.
+// kubenet uses the CNI bridge plugin, which stores allocated ips on file. Each
+// file created under defaultIPAMDir has the format: ip/container-hash. So this
+// routine looks for hashes that are not reported by the currently running docker,
+// and invokes DelNetwork on each one. Note that this will only work for the
+// current CNI bridge plugin, because we have no way of finding the NetNs.
+func (plugin *kubenetNetworkPlugin) ipamGarbageCollection() {
+	glog.V(2).Infof("Starting IP garbage collection")
+
+	ipamDir := filepath.Join(defaultIPAMDir, KubenetPluginName)
+	files, err := ioutil.ReadDir(ipamDir)
+	if err != nil {
+		glog.Errorf("Failed to list files in %q: %v", ipamDir, err)
+		return
+	}
+
+	// gather containerIDs for allocated ips
+	ipContainerIdMap := make(map[string]string)
+	for _, file := range files {
+		// skip non checkpoint file
+		if ip := net.ParseIP(file.Name()); ip == nil {
+			continue
+		}
+
+		content, err := ioutil.ReadFile(filepath.Join(ipamDir, file.Name()))
+		if err != nil {
+			glog.Errorf("Failed to read file %v: %v", file, err)
+		}
+		ipContainerIdMap[file.Name()] = strings.TrimSpace(string(content))
+	}
+
+	// gather infra container IDs of current running Pods
+	runningContainerIDs := utilsets.String{}
+	pods, err := plugin.getNonExitedPods()
+	if err != nil {
+		glog.Errorf("Failed to get pods: %v", err)
+		return
+	}
+	for _, pod := range pods {
+		containerID, err := plugin.host.GetRuntime().GetPodContainerID(pod)
+		if err != nil {
+			glog.Warningf("Failed to get infra containerID of %q/%q: %v", pod.Namespace, pod.Name, err)
+			continue
+		}
+
+		runningContainerIDs.Insert(strings.TrimSpace(containerID.ID))
+	}
+
+	// release leaked ips
+	for ip, containerID := range ipContainerIdMap {
+		// if the container is not running, release IP
+		if runningContainerIDs.Has(containerID) {
+			continue
+		}
+		// CNI requires all config to be presented, although only containerID is needed in this case
+		rt := &libcni.RuntimeConf{
+			ContainerID: containerID,
+			IfName:      network.DefaultInterfaceName,
+			// TODO: How do we find the NetNs of an exited container? docker inspect
+			// doesn't show us the pid, so we probably need to checkpoint
+			NetNS: "",
+		}
+
+		glog.V(2).Infof("Releasing IP %q allocated to %q.", ip, containerID)
+		// CNI bridge plugin should try to release IP and then return
+		if err := plugin.cniConfig.DelNetwork(plugin.netConfig, rt); err != nil {
+			glog.Errorf("Error while releasing IP: %v", err)
+		}
+	}
+}
+
+// podIsExited returns true if the pod is exited (all containers inside are exited).
+func podIsExited(p *kubecontainer.Pod) bool {
+	for _, c := range p.Containers {
+		if c.State != kubecontainer.ContainerStateExited {
+			return false
+		}
+	}
+	for _, c := range p.Sandboxes {
+		if c.State != kubecontainer.ContainerStateExited {
+			return false
+		}
+	}
+	return true
+}
+
+func (plugin *kubenetNetworkPlugin) buildCNIRuntimeConf(ifName string, id kubecontainer.ContainerID, needNetNs bool) (*libcni.RuntimeConf, error) {
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if needNetNs && err != nil {
+		glog.Errorf("Kubenet failed to retrieve network namespace path: %v", err)
+	}
+
+	return &libcni.RuntimeConf{
+		ContainerID: id.ID,
+		NetNS:       netnsPath,
+		IfName:      ifName,
+	}, nil
+}
+
+func (plugin *kubenetNetworkPlugin) addContainerToNetwork(config *libcni.NetworkConfig, ifName, namespace, name string, id kubecontainer.ContainerID) (cnitypes.Result, error) {
+	rt, err := plugin.buildCNIRuntimeConf(ifName, id, true)
+	if err != nil {
+		return nil, fmt.Errorf("Error building CNI config: %v", err)
+	}
+
+	glog.V(3).Infof("Adding %s/%s to '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt)
+	// The network plugin can take up to 3 seconds to execute,
+	// so yield the lock while it runs.
+	plugin.mu.Unlock()
+	res, err := plugin.cniConfig.AddNetwork(config, rt)
+	plugin.mu.Lock()
+	if err != nil {
+		return nil, fmt.Errorf("Error adding container to network: %v", err)
+	}
+	return res, nil
+}
+
+func (plugin *kubenetNetworkPlugin) delContainerFromNetwork(config *libcni.NetworkConfig, ifName, namespace, name string, id kubecontainer.ContainerID) error {
+	rt, err := plugin.buildCNIRuntimeConf(ifName, id, false)
+	if err != nil {
+		return fmt.Errorf("Error building CNI config: %v", err)
+	}
+
+	glog.V(3).Infof("Removing %s/%s from '%s' with CNI '%s' plugin and runtime: %+v", namespace, name, config.Network.Name, config.Network.Type, rt)
+	if err := plugin.cniConfig.DelNetwork(config, rt); err != nil {
+		return fmt.Errorf("Error removing container from network: %v", err)
+	}
+	return nil
+}
+
+// shaper retrieves the bandwidth shaper and, if it hasn't been fetched before,
+// initializes it and ensures the bridge is appropriately configured
+// This function should only be called while holding the `plugin.mu` lock
+func (plugin *kubenetNetworkPlugin) shaper() bandwidth.BandwidthShaper {
+	if plugin.bandwidthShaper == nil {
+		plugin.bandwidthShaper = bandwidth.NewTCShaper(BridgeName)
+		plugin.bandwidthShaper.ReconcileInterface()
+	}
+	return plugin.bandwidthShaper
+}
+
+//TODO: make this into a goroutine and rectify the dedup rules periodically
+func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareAddr) {
+	if plugin.ebtables == nil {
+		plugin.ebtables = utilebtables.New(plugin.execer)
+		glog.V(3).Infof("Flushing dedup chain")
+		if err := plugin.ebtables.FlushChain(utilebtables.TableFilter, dedupChain); err != nil {
+			glog.Errorf("Failed to flush dedup chain: %v", err)
+		}
+	}
+	_, err := plugin.ebtables.GetVersion()
+	if err != nil {
+		glog.Warningf("Failed to get ebtables version. Skip syncing ebtables dedup rules: %v", err)
+		return
+	}
+
+	glog.V(3).Infof("Filtering packets with ebtables on mac address: %v, gateway: %v, pod CIDR: %v", macAddr.String(), plugin.gateway.String(), plugin.podCidr)
+	_, err = plugin.ebtables.EnsureChain(utilebtables.TableFilter, dedupChain)
+	if err != nil {
+		glog.Errorf("Failed to ensure %v chain %v", utilebtables.TableFilter, dedupChain)
+		return
+	}
+
+	_, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, utilebtables.ChainOutput, "-j", string(dedupChain))
+	if err != nil {
+		glog.Errorf("Failed to ensure %v chain %v jump to %v chain: %v", utilebtables.TableFilter, utilebtables.ChainOutput, dedupChain, err)
+		return
+	}
+
+	commonArgs := []string{"-p", "IPv4", "-s", macAddr.String(), "-o", "veth+"}
+	_, err = plugin.ebtables.EnsureRule(utilebtables.Prepend, utilebtables.TableFilter, dedupChain, append(commonArgs, "--ip-src", plugin.gateway.String(), "-j", "ACCEPT")...)
+	if err != nil {
+		glog.Errorf("Failed to ensure packets from cbr0 gateway to be accepted")
+		return
+
+	}
+	_, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, dedupChain, append(commonArgs, "--ip-src", plugin.podCidr, "-j", "DROP")...)
+	if err != nil {
+		glog.Errorf("Failed to ensure packets from podCidr but has mac address of cbr0 to get dropped.")
+		return
+	}
+}
+
+// disableContainerDAD disables duplicate address detection in the container.
+// DAD has a negative affect on pod creation latency, since we have to wait
+// a second or more for the addresses to leave the "tentative" state. Since
+// we're sure there won't be an address conflict (since we manage them manually),
+// this is safe. See issue 54651.
+//
+// This sets net.ipv6.conf.default.dad_transmits to 0. It must be run *before*
+// the CNI plugins are run.
+func (plugin *kubenetNetworkPlugin) disableContainerDAD(id kubecontainer.ContainerID) error {
+	key := "net/ipv6/conf/default/dad_transmits"
+
+	sysctlBin, err := plugin.execer.LookPath("sysctl")
+	if err != nil {
+		return fmt.Errorf("Could not find sysctl binary: %s", err)
+	}
+
+	netnsPath, err := plugin.host.GetNetNS(id.ID)
+	if err != nil {
+		return fmt.Errorf("Failed to get netns: %v", err)
+	}
+	if netnsPath == "" {
+		return fmt.Errorf("Pod has no network namespace")
+	}
+
+	// If the sysctl doesn't exist, it means ipv6 is disabled; log and move on
+	if _, err := plugin.sysctl.GetSysctl(key); err != nil {
+		return fmt.Errorf("Ipv6 not enabled: %v", err)
+	}
+
+	output, err := plugin.execer.Command(plugin.nsenterPath,
+		fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
+		sysctlBin, "-w", fmt.Sprintf("%s=%s", key, "0"),
+	).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("Failed to write sysctl: output: %s error: %s",
+			output, err)
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/kubenet/kubenet_linux_test.go

@@ -0,0 +1,262 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubenet
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	"testing"
+
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	"k8s.io/kubernetes/pkg/kubelet/network/cni/testing"
+	hostporttest "k8s.io/kubernetes/pkg/kubelet/network/hostport/testing"
+	nettest "k8s.io/kubernetes/pkg/kubelet/network/testing"
+	"k8s.io/kubernetes/pkg/util/bandwidth"
+	ipttest "k8s.io/kubernetes/pkg/util/iptables/testing"
+	sysctltest "k8s.io/kubernetes/pkg/util/sysctl/testing"
+	"k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+// test it fulfills the NetworkPlugin interface
+var _ network.NetworkPlugin = &kubenetNetworkPlugin{}
+
+func newFakeKubenetPlugin(initMap map[kubecontainer.ContainerID]string, execer exec.Interface, host network.Host) *kubenetNetworkPlugin {
+	return &kubenetNetworkPlugin{
+		podIPs: initMap,
+		execer: execer,
+		mtu:    1460,
+		host:   host,
+	}
+}
+
+func TestGetPodNetworkStatus(t *testing.T) {
+	podIPMap := make(map[kubecontainer.ContainerID]string)
+	podIPMap[kubecontainer.ContainerID{ID: "1"}] = "10.245.0.2"
+	podIPMap[kubecontainer.ContainerID{ID: "2"}] = "10.245.0.3"
+
+	testCases := []struct {
+		id          string
+		expectError bool
+		expectIP    string
+	}{
+		//in podCIDR map
+		{
+			"1",
+			false,
+			"10.245.0.2",
+		},
+		{
+			"2",
+			false,
+			"10.245.0.3",
+		},
+		//not in podCIDR map
+		{
+			"3",
+			true,
+			"",
+		},
+		//TODO: add test cases for retrieving ip inside container network namespace
+	}
+
+	fakeCmds := make([]fakeexec.FakeCommandAction, 0)
+	for _, t := range testCases {
+		// the fake commands return the IP from the given index, or an error
+		fCmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				func() ([]byte, error) {
+					ip, ok := podIPMap[kubecontainer.ContainerID{ID: t.id}]
+					if !ok {
+						return nil, fmt.Errorf("Pod IP %q not found", t.id)
+					}
+					return []byte(ip), nil
+				},
+			},
+		}
+		fakeCmds = append(fakeCmds, func(cmd string, args ...string) exec.Cmd {
+			return fakeexec.InitFakeCmd(&fCmd, cmd, args...)
+		})
+	}
+	fexec := fakeexec.FakeExec{
+		CommandScript: fakeCmds,
+		LookPathFunc: func(file string) (string, error) {
+			return fmt.Sprintf("/fake-bin/%s", file), nil
+		},
+	}
+
+	fhost := nettest.NewFakeHost(nil)
+	fakeKubenet := newFakeKubenetPlugin(podIPMap, &fexec, fhost)
+
+	for i, tc := range testCases {
+		out, err := fakeKubenet.GetPodNetworkStatus("", "", kubecontainer.ContainerID{ID: tc.id})
+		if tc.expectError {
+			if err == nil {
+				t.Errorf("Test case %d expects error but got none", i)
+			}
+			continue
+		} else {
+			if err != nil {
+				t.Errorf("Test case %d expects error but got error: %v", i, err)
+			}
+		}
+		if tc.expectIP != out.IP.String() {
+			t.Errorf("Test case %d expects ip %s but got %s", i, tc.expectIP, out.IP.String())
+		}
+	}
+}
+
+// TestTeardownCallsShaper tests that a `TearDown` call does call
+// `shaper.Reset`
+func TestTeardownCallsShaper(t *testing.T) {
+	fexec := &fakeexec.FakeExec{
+		CommandScript: []fakeexec.FakeCommandAction{},
+		LookPathFunc: func(file string) (string, error) {
+			return fmt.Sprintf("/fake-bin/%s", file), nil
+		},
+	}
+	fhost := nettest.NewFakeHost(nil)
+	fshaper := &bandwidth.FakeShaper{}
+	mockcni := &mock_cni.MockCNI{}
+	kubenet := newFakeKubenetPlugin(map[kubecontainer.ContainerID]string{}, fexec, fhost)
+	kubenet.cniConfig = mockcni
+	kubenet.iptables = ipttest.NewFake()
+	kubenet.bandwidthShaper = fshaper
+	kubenet.hostportSyncer = hostporttest.NewFakeHostportSyncer()
+
+	mockcni.On("DelNetwork", mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil)
+
+	details := make(map[string]interface{})
+	details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = "10.0.0.1/24"
+	kubenet.Event(network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE, details)
+
+	existingContainerID := kubecontainer.BuildContainerID("docker", "123")
+	kubenet.podIPs[existingContainerID] = "10.0.0.1"
+
+	if err := kubenet.TearDownPod("namespace", "name", existingContainerID); err != nil {
+		t.Fatalf("Unexpected error in TearDownPod: %v", err)
+	}
+	assert.Equal(t, []string{"10.0.0.1/32"}, fshaper.ResetCIDRs, "shaper.Reset should have been called")
+
+	mockcni.AssertExpectations(t)
+}
+
+// TestInit tests that a `Init` call with an MTU sets the MTU
+func TestInit_MTU(t *testing.T) {
+	var fakeCmds []fakeexec.FakeCommandAction
+	{
+		// modprobe br-netfilter
+		fCmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				func() ([]byte, error) {
+					return make([]byte, 0), nil
+				},
+			},
+		}
+		fakeCmds = append(fakeCmds, func(cmd string, args ...string) exec.Cmd {
+			return fakeexec.InitFakeCmd(&fCmd, cmd, args...)
+		})
+	}
+
+	fexec := &fakeexec.FakeExec{
+		CommandScript: fakeCmds,
+		LookPathFunc: func(file string) (string, error) {
+			return fmt.Sprintf("/fake-bin/%s", file), nil
+		},
+	}
+
+	fhost := nettest.NewFakeHost(nil)
+	kubenet := newFakeKubenetPlugin(map[kubecontainer.ContainerID]string{}, fexec, fhost)
+	kubenet.iptables = ipttest.NewFake()
+
+	sysctl := sysctltest.NewFake()
+	sysctl.Settings["net/bridge/bridge-nf-call-iptables"] = 0
+	kubenet.sysctl = sysctl
+
+	if err := kubenet.Init(nettest.NewFakeHost(nil), kubeletconfig.HairpinNone, "10.0.0.0/8", 1234); err != nil {
+		t.Fatalf("Unexpected error in Init: %v", err)
+	}
+	assert.Equal(t, 1234, kubenet.mtu, "kubenet.mtu should have been set")
+	assert.Equal(t, 1, sysctl.Settings["net/bridge/bridge-nf-call-iptables"], "net/bridge/bridge-nf-call-iptables sysctl should have been set")
+}
+
+// TestInvocationWithoutRuntime invokes the plugin without a runtime.
+// This is how kubenet is invoked from the cri.
+func TestTearDownWithoutRuntime(t *testing.T) {
+	testCases := []struct {
+		podCIDR         string
+		ip              string
+		expectedGateway string
+	}{
+		{
+			podCIDR:         "10.0.0.1/24",
+			ip:              "10.0.0.1",
+			expectedGateway: "10.0.0.1",
+		},
+		{
+			podCIDR:         "2001:beef::1/48",
+			ip:              "2001:beef::1",
+			expectedGateway: "2001:beef::1",
+		},
+	}
+	for _, tc := range testCases {
+		fhost := nettest.NewFakeHost(nil)
+		fhost.Legacy = false
+		fhost.Runtime = nil
+		mockcni := &mock_cni.MockCNI{}
+
+		fexec := &fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{},
+			LookPathFunc: func(file string) (string, error) {
+				return fmt.Sprintf("/fake-bin/%s", file), nil
+			},
+		}
+
+		kubenet := newFakeKubenetPlugin(map[kubecontainer.ContainerID]string{}, fexec, fhost)
+		kubenet.cniConfig = mockcni
+		kubenet.iptables = ipttest.NewFake()
+
+		details := make(map[string]interface{})
+		details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = tc.podCIDR
+		kubenet.Event(network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE, details)
+
+		if kubenet.gateway.String() != tc.expectedGateway {
+			t.Errorf("generated gateway: %q, expecting: %q", kubenet.gateway.String(), tc.expectedGateway)
+		}
+		if kubenet.podCidr != tc.podCIDR {
+			t.Errorf("generated podCidr: %q, expecting: %q", kubenet.podCidr, tc.podCIDR)
+		}
+		existingContainerID := kubecontainer.BuildContainerID("docker", "123")
+		kubenet.podIPs[existingContainerID] = tc.ip
+
+		mockcni.On("DelNetwork", mock.AnythingOfType("*libcni.NetworkConfig"), mock.AnythingOfType("*libcni.RuntimeConf")).Return(nil)
+
+		if err := kubenet.TearDownPod("namespace", "name", existingContainerID); err != nil {
+			t.Fatalf("Unexpected error in TearDownPod: %v", err)
+		}
+		// Assert that the CNI DelNetwork made it through and we didn't crash
+		// without a runtime.
+		mockcni.AssertExpectations(t)
+	}
+}
+
+//TODO: add unit test for each implementation of network plugin interface

vendor/k8s.io/kubernetes/pkg/kubelet/network/kubenet/kubenet_unsupported.go

@@ -0,0 +1,55 @@
+// +build !linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubenet
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+type kubenetNetworkPlugin struct {
+	network.NoopNetworkPlugin
+}
+
+func NewPlugin(networkPluginDir string) network.NetworkPlugin {
+	return &kubenetNetworkPlugin{}
+}
+
+func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	return fmt.Errorf("Kubenet is not supported in this build")
+}
+
+func (plugin *kubenetNetworkPlugin) Name() string {
+	return "kubenet"
+}
+
+func (plugin *kubenetNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	return fmt.Errorf("Kubenet is not supported in this build")
+}
+
+func (plugin *kubenetNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.ContainerID) error {
+	return fmt.Errorf("Kubenet is not supported in this build")
+}
+
+func (plugin *kubenetNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	return nil, fmt.Errorf("Kubenet is not supported in this build")
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/metrics/BUILD

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metrics.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/metrics",
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/prometheus/client_golang/prometheus:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/metrics/metrics.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// NetworkPluginOperationsKey is the key for operation count metrics.
+	NetworkPluginOperationsKey = "network_plugin_operations"
+	// NetworkPluginOperationsLatencyKey is the key for the operation latency metrics.
+	NetworkPluginOperationsLatencyKey = "network_plugin_operations_latency_microseconds"
+
+	// Keep the "kubelet" subsystem for backward compatibility.
+	kubeletSubsystem = "kubelet"
+)
+
+var (
+	// NetworkPluginOperationsLatency collects operation latency numbers by operation
+	// type.
+	NetworkPluginOperationsLatency = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Subsystem: kubeletSubsystem,
+			Name:      NetworkPluginOperationsLatencyKey,
+			Help:      "Latency in microseconds of network plugin operations. Broken down by operation type.",
+		},
+		[]string{"operation_type"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(NetworkPluginOperationsLatency)
+	})
+}
+
+// SinceInMicroseconds gets the time since the specified start in microseconds.
+func SinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/network.go

@@ -0,0 +1,24 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+// TODO: Consider making this value configurable.
+const DefaultInterfaceName = "eth0"
+
+// UseDefaultMTU is a marker value that indicates the plugin should determine its own MTU
+// It is the zero value, so a non-initialized value will mean "UseDefault"
+const UseDefaultMTU = 0

vendor/k8s.io/kubernetes/pkg/kubelet/network/plugins.go

@@ -0,0 +1,432 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package network
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilsets "k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
+	"k8s.io/kubernetes/pkg/kubelet/network/metrics"
+	utilsysctl "k8s.io/kubernetes/pkg/util/sysctl"
+	utilexec "k8s.io/utils/exec"
+)
+
+const DefaultPluginName = "kubernetes.io/no-op"
+
+// Called when the node's Pod CIDR is known when using the
+// controller manager's --allocate-node-cidrs=true option
+const NET_PLUGIN_EVENT_POD_CIDR_CHANGE = "pod-cidr-change"
+const NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR = "pod-cidr"
+
+// Plugin is an interface to network plugins for the kubelet
+type NetworkPlugin interface {
+	// Init initializes the plugin.  This will be called exactly once
+	// before any other methods are called.
+	Init(host Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error
+
+	// Called on various events like:
+	// NET_PLUGIN_EVENT_POD_CIDR_CHANGE
+	Event(name string, details map[string]interface{})
+
+	// Name returns the plugin's name. This will be used when searching
+	// for a plugin by name, e.g.
+	Name() string
+
+	// Returns a set of NET_PLUGIN_CAPABILITY_*
+	Capabilities() utilsets.Int
+
+	// SetUpPod is the method called after the infra container of
+	// the pod has been created but before the other containers of the
+	// pod are launched.
+	SetUpPod(namespace string, name string, podSandboxID kubecontainer.ContainerID, annotations map[string]string) error
+
+	// TearDownPod is the method called before a pod's infra container will be deleted
+	TearDownPod(namespace string, name string, podSandboxID kubecontainer.ContainerID) error
+
+	// GetPodNetworkStatus is the method called to obtain the ipv4 or ipv6 addresses of the container
+	GetPodNetworkStatus(namespace string, name string, podSandboxID kubecontainer.ContainerID) (*PodNetworkStatus, error)
+
+	// Status returns error if the network plugin is in error state
+	Status() error
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// PodNetworkStatus stores the network status of a pod (currently just the primary IP address)
+// This struct represents version "v1beta1"
+type PodNetworkStatus struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// IP is the primary ipv4/ipv6 address of the pod. Among other things it is the address that -
+	//   - kube expects to be reachable across the cluster
+	//   - service endpoints are constructed with
+	//   - will be reported in the PodStatus.PodIP field (will override the IP reported by docker)
+	IP net.IP `json:"ip" description:"Primary IP address of the pod"`
+}
+
+// LegacyHost implements the methods required by network plugins that
+// were directly invoked by the kubelet. Implementations of this interface
+// that do not wish to support these features can simply return false
+// to SupportsLegacyFeatures.
+type LegacyHost interface {
+	// Get the pod structure by its name, namespace
+	// Only used for hostport management and bw shaping
+	GetPodByName(namespace, name string) (*v1.Pod, bool)
+
+	// GetKubeClient returns a client interface
+	// Only used in testing
+	GetKubeClient() clientset.Interface
+
+	// GetContainerRuntime returns the container runtime that implements the containers (e.g. docker/rkt)
+	// Only used for hostport management
+	GetRuntime() kubecontainer.Runtime
+
+	// SupportsLegacyFeatures returns true if the network host support GetPodByName, KubeClient interface and kubelet
+	// runtime interface. These interfaces will no longer be implemented by CRI shims.
+	// This function helps network plugins to choose their behavior based on runtime.
+	SupportsLegacyFeatures() bool
+}
+
+// Host is an interface that plugins can use to access the kubelet.
+// TODO(#35457): get rid of this backchannel to the kubelet. The scope of
+// the back channel is restricted to host-ports/testing, and restricted
+// to kubenet. No other network plugin wrapper needs it. Other plugins
+// only require a way to access namespace information and port mapping
+// information , which they can do directly through the embedded interfaces.
+type Host interface {
+	// NamespaceGetter is a getter for sandbox namespace information.
+	NamespaceGetter
+
+	// PortMappingGetter is a getter for sandbox port mapping information.
+	PortMappingGetter
+
+	// LegacyHost contains methods that trap back into the Kubelet. Dependence
+	// *do not* add more dependencies in this interface. In a post-cri world,
+	// network plugins will be invoked by the runtime shim, and should only
+	// require GetNetNS and GetPodPortMappings.
+	LegacyHost
+}
+
+// NamespaceGetter is an interface to retrieve namespace information for a given
+// podSandboxID. Typically implemented by runtime shims that are closely coupled to
+// CNI plugin wrappers like kubenet.
+type NamespaceGetter interface {
+	// GetNetNS returns network namespace information for the given containerID.
+	// Runtimes should *never* return an empty namespace and nil error for
+	// a container; if error is nil then the namespace string must be valid.
+	GetNetNS(containerID string) (string, error)
+}
+
+// PortMappingGetter is an interface to retrieve port mapping information for a given
+// podSandboxID. Typically implemented by runtime shims that are closely coupled to
+// CNI plugin wrappers like kubenet.
+type PortMappingGetter interface {
+	// GetPodPortMappings returns sandbox port mappings information.
+	GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error)
+}
+
+// InitNetworkPlugin inits the plugin that matches networkPluginName. Plugins must have unique names.
+func InitNetworkPlugin(plugins []NetworkPlugin, networkPluginName string, host Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) (NetworkPlugin, error) {
+	if networkPluginName == "" {
+		// default to the no_op plugin
+		plug := &NoopNetworkPlugin{}
+		plug.Sysctl = utilsysctl.New()
+		if err := plug.Init(host, hairpinMode, nonMasqueradeCIDR, mtu); err != nil {
+			return nil, err
+		}
+		return plug, nil
+	}
+
+	pluginMap := map[string]NetworkPlugin{}
+
+	allErrs := []error{}
+	for _, plugin := range plugins {
+		name := plugin.Name()
+		if errs := validation.IsQualifiedName(name); len(errs) != 0 {
+			allErrs = append(allErrs, fmt.Errorf("network plugin has invalid name: %q: %s", name, strings.Join(errs, ";")))
+			continue
+		}
+
+		if _, found := pluginMap[name]; found {
+			allErrs = append(allErrs, fmt.Errorf("network plugin %q was registered more than once", name))
+			continue
+		}
+		pluginMap[name] = plugin
+	}
+
+	chosenPlugin := pluginMap[networkPluginName]
+	if chosenPlugin != nil {
+		err := chosenPlugin.Init(host, hairpinMode, nonMasqueradeCIDR, mtu)
+		if err != nil {
+			allErrs = append(allErrs, fmt.Errorf("Network plugin %q failed init: %v", networkPluginName, err))
+		} else {
+			glog.V(1).Infof("Loaded network plugin %q", networkPluginName)
+		}
+	} else {
+		allErrs = append(allErrs, fmt.Errorf("Network plugin %q not found.", networkPluginName))
+	}
+
+	return chosenPlugin, utilerrors.NewAggregate(allErrs)
+}
+
+func UnescapePluginName(in string) string {
+	return strings.Replace(in, "~", "/", -1)
+}
+
+type NoopNetworkPlugin struct {
+	Sysctl utilsysctl.Interface
+}
+
+const sysctlBridgeCallIPTables = "net/bridge/bridge-nf-call-iptables"
+const sysctlBridgeCallIP6Tables = "net/bridge/bridge-nf-call-ip6tables"
+
+func (plugin *NoopNetworkPlugin) Init(host Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	// Set bridge-nf-call-iptables=1 to maintain compatibility with older
+	// kubernetes versions to ensure the iptables-based kube proxy functions
+	// correctly.  Other plugins are responsible for setting this correctly
+	// depending on whether or not they connect containers to Linux bridges
+	// or use some other mechanism (ie, SDN vswitch).
+
+	// Ensure the netfilter module is loaded on kernel >= 3.18; previously
+	// it was built-in.
+	utilexec.New().Command("modprobe", "br-netfilter").CombinedOutput()
+	if err := plugin.Sysctl.SetSysctl(sysctlBridgeCallIPTables, 1); err != nil {
+		glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIPTables, err)
+	}
+	if val, err := plugin.Sysctl.GetSysctl(sysctlBridgeCallIP6Tables); err == nil {
+		if val != 1 {
+			if err = plugin.Sysctl.SetSysctl(sysctlBridgeCallIP6Tables, 1); err != nil {
+				glog.Warningf("can't set sysctl %s: %v", sysctlBridgeCallIP6Tables, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (plugin *NoopNetworkPlugin) Event(name string, details map[string]interface{}) {
+}
+
+func (plugin *NoopNetworkPlugin) Name() string {
+	return DefaultPluginName
+}
+
+func (plugin *NoopNetworkPlugin) Capabilities() utilsets.Int {
+	return utilsets.NewInt()
+}
+
+func (plugin *NoopNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	return nil
+}
+
+func (plugin *NoopNetworkPlugin) TearDownPod(namespace string, name string, id kubecontainer.ContainerID) error {
+	return nil
+}
+
+func (plugin *NoopNetworkPlugin) GetPodNetworkStatus(namespace string, name string, id kubecontainer.ContainerID) (*PodNetworkStatus, error) {
+	return nil, nil
+}
+
+func (plugin *NoopNetworkPlugin) Status() error {
+	return nil
+}
+
+func getOnePodIP(execer utilexec.Interface, nsenterPath, netnsPath, interfaceName, addrType string) (net.IP, error) {
+	// Try to retrieve ip inside container network namespace
+	output, err := execer.Command(nsenterPath, fmt.Sprintf("--net=%s", netnsPath), "-F", "--",
+		"ip", "-o", addrType, "addr", "show", "dev", interfaceName, "scope", "global").CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("Unexpected command output %s with error: %v", output, err)
+	}
+
+	lines := strings.Split(string(output), "\n")
+	if len(lines) < 1 {
+		return nil, fmt.Errorf("Unexpected command output %s", output)
+	}
+	fields := strings.Fields(lines[0])
+	if len(fields) < 4 {
+		return nil, fmt.Errorf("Unexpected address output %s ", lines[0])
+	}
+	ip, _, err := net.ParseCIDR(fields[3])
+	if err != nil {
+		return nil, fmt.Errorf("CNI failed to parse ip from output %s due to %v", output, err)
+	}
+
+	return ip, nil
+}
+
+// GetPodIP gets the IP of the pod by inspecting the network info inside the pod's network namespace.
+func GetPodIP(execer utilexec.Interface, nsenterPath, netnsPath, interfaceName string) (net.IP, error) {
+	ip, err := getOnePodIP(execer, nsenterPath, netnsPath, interfaceName, "-4")
+	if err != nil {
+		// Fall back to IPv6 address if no IPv4 address is present
+		ip, err = getOnePodIP(execer, nsenterPath, netnsPath, interfaceName, "-6")
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return ip, nil
+}
+
+type NoopPortMappingGetter struct{}
+
+func (*NoopPortMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) {
+	return nil, nil
+}
+
+// The PluginManager wraps a kubelet network plugin and provides synchronization
+// for a given pod's network operations.  Each pod's setup/teardown/status operations
+// are synchronized against each other, but network operations of other pods can
+// proceed in parallel.
+type PluginManager struct {
+	// Network plugin being wrapped
+	plugin NetworkPlugin
+
+	// Pod list and lock
+	podsLock sync.Mutex
+	pods     map[string]*podLock
+}
+
+func NewPluginManager(plugin NetworkPlugin) *PluginManager {
+	metrics.Register()
+	return &PluginManager{
+		plugin: plugin,
+		pods:   make(map[string]*podLock),
+	}
+}
+
+func (pm *PluginManager) PluginName() string {
+	return pm.plugin.Name()
+}
+
+func (pm *PluginManager) Event(name string, details map[string]interface{}) {
+	pm.plugin.Event(name, details)
+}
+
+func (pm *PluginManager) Status() error {
+	return pm.plugin.Status()
+}
+
+type podLock struct {
+	// Count of in-flight operations for this pod; when this reaches zero
+	// the lock can be removed from the pod map
+	refcount uint
+
+	// Lock to synchronize operations for this specific pod
+	mu sync.Mutex
+}
+
+// Lock network operations for a specific pod.  If that pod is not yet in
+// the pod map, it will be added.  The reference count for the pod will
+// be increased.
+func (pm *PluginManager) podLock(fullPodName string) *sync.Mutex {
+	pm.podsLock.Lock()
+	defer pm.podsLock.Unlock()
+
+	lock, ok := pm.pods[fullPodName]
+	if !ok {
+		lock = &podLock{}
+		pm.pods[fullPodName] = lock
+	}
+	lock.refcount++
+	return &lock.mu
+}
+
+// Unlock network operations for a specific pod.  The reference count for the
+// pod will be decreased.  If the reference count reaches zero, the pod will be
+// removed from the pod map.
+func (pm *PluginManager) podUnlock(fullPodName string) {
+	pm.podsLock.Lock()
+	defer pm.podsLock.Unlock()
+
+	lock, ok := pm.pods[fullPodName]
+	if !ok {
+		glog.Warningf("Unbalanced pod lock unref for %s", fullPodName)
+		return
+	} else if lock.refcount == 0 {
+		// This should never ever happen, but handle it anyway
+		delete(pm.pods, fullPodName)
+		glog.Warningf("Pod lock for %s still in map with zero refcount", fullPodName)
+		return
+	}
+	lock.refcount--
+	lock.mu.Unlock()
+	if lock.refcount == 0 {
+		delete(pm.pods, fullPodName)
+	}
+}
+
+// recordOperation records operation and duration
+func recordOperation(operation string, start time.Time) {
+	metrics.NetworkPluginOperationsLatency.WithLabelValues(operation).Observe(metrics.SinceInMicroseconds(start))
+}
+
+func (pm *PluginManager) GetPodNetworkStatus(podNamespace, podName string, id kubecontainer.ContainerID) (*PodNetworkStatus, error) {
+	defer recordOperation("get_pod_network_status", time.Now())
+	fullPodName := kubecontainer.BuildPodFullName(podName, podNamespace)
+	pm.podLock(fullPodName).Lock()
+	defer pm.podUnlock(fullPodName)
+
+	netStatus, err := pm.plugin.GetPodNetworkStatus(podNamespace, podName, id)
+	if err != nil {
+		return nil, fmt.Errorf("NetworkPlugin %s failed on the status hook for pod %q: %v", pm.plugin.Name(), fullPodName, err)
+	}
+
+	return netStatus, nil
+}
+
+func (pm *PluginManager) SetUpPod(podNamespace, podName string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	defer recordOperation("set_up_pod", time.Now())
+	fullPodName := kubecontainer.BuildPodFullName(podName, podNamespace)
+	pm.podLock(fullPodName).Lock()
+	defer pm.podUnlock(fullPodName)
+
+	glog.V(3).Infof("Calling network plugin %s to set up pod %q", pm.plugin.Name(), fullPodName)
+	if err := pm.plugin.SetUpPod(podNamespace, podName, id, annotations); err != nil {
+		return fmt.Errorf("NetworkPlugin %s failed to set up pod %q network: %v", pm.plugin.Name(), fullPodName, err)
+	}
+
+	return nil
+}
+
+func (pm *PluginManager) TearDownPod(podNamespace, podName string, id kubecontainer.ContainerID) error {
+	defer recordOperation("tear_down_pod", time.Now())
+	fullPodName := kubecontainer.BuildPodFullName(podName, podNamespace)
+	pm.podLock(fullPodName).Lock()
+	defer pm.podUnlock(fullPodName)
+
+	glog.V(3).Infof("Calling network plugin %s to tear down pod %q", pm.plugin.Name(), fullPodName)
+	if err := pm.plugin.TearDownPod(podNamespace, podName, id); err != nil {
+		return fmt.Errorf("NetworkPlugin %s failed to teardown pod %q network: %v", pm.plugin.Name(), fullPodName, err)
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/testing/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "fake_host.go",
+        "mock_network_plugin.go",
+    ],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/testing",
+    deps = [
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/container/testing:go_default_library",
+        "//pkg/kubelet/network:go_default_library",
+        "//pkg/kubelet/network/hostport:go_default_library",
+        "//vendor/github.com/golang/mock/gomock:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["plugins_test.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/network/testing",
+    library = ":go_default_library",
+    deps = [
+        "//pkg/kubelet/apis/kubeletconfig:go_default_library",
+        "//pkg/kubelet/container:go_default_library",
+        "//pkg/kubelet/network:go_default_library",
+        "//pkg/util/sysctl/testing:go_default_library",
+        "//vendor/github.com/golang/mock/gomock:go_default_library",
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/pkg/kubelet/network/testing/fake_host.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+// helper for testing plugins
+// a fake host is created here that can be used by plugins for testing
+
+import (
+	"k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
+	"k8s.io/kubernetes/pkg/kubelet/network/hostport"
+)
+
+type fakeNetworkHost struct {
+	fakeNamespaceGetter
+	FakePortMappingGetter
+	kubeClient clientset.Interface
+	Legacy     bool
+	Runtime    *containertest.FakeRuntime
+}
+
+func NewFakeHost(kubeClient clientset.Interface) *fakeNetworkHost {
+	host := &fakeNetworkHost{kubeClient: kubeClient, Legacy: true, Runtime: &containertest.FakeRuntime{}}
+	return host
+}
+
+func (fnh *fakeNetworkHost) GetPodByName(name, namespace string) (*v1.Pod, bool) {
+	return nil, false
+}
+
+func (fnh *fakeNetworkHost) GetKubeClient() clientset.Interface {
+	return nil
+}
+
+func (nh *fakeNetworkHost) GetRuntime() kubecontainer.Runtime {
+	return nh.Runtime
+}
+
+func (nh *fakeNetworkHost) SupportsLegacyFeatures() bool {
+	return nh.Legacy
+}
+
+type fakeNamespaceGetter struct {
+	ns string
+}
+
+func (nh *fakeNamespaceGetter) GetNetNS(containerID string) (string, error) {
+	return nh.ns, nil
+}
+
+type FakePortMappingGetter struct {
+	PortMaps map[string][]*hostport.PortMapping
+}
+
+func (pm *FakePortMappingGetter) GetPodPortMappings(containerID string) ([]*hostport.PortMapping, error) {
+	return pm.PortMaps[containerID], nil
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/testing/mock_network_plugin.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Generated code, generated via: `mockgen k8s.io/kubernetes/pkg/kubelet/network NetworkPlugin > $GOPATH/src/k8s.io/kubernetes/pkg/kubelet/network/testing/mock_network_plugin.go`
+// Edited by hand for boilerplate and gofmt.
+// TODO, this should be autogenerated/autoupdated by scripts.
+
+package testing
+
+import (
+	gomock "github.com/golang/mock/gomock"
+	sets "k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	container "k8s.io/kubernetes/pkg/kubelet/container"
+	network "k8s.io/kubernetes/pkg/kubelet/network"
+)
+
+// Mock of NetworkPlugin interface
+type MockNetworkPlugin struct {
+	ctrl     *gomock.Controller
+	recorder *_MockNetworkPluginRecorder
+}
+
+// Recorder for MockNetworkPlugin (not exported)
+type _MockNetworkPluginRecorder struct {
+	mock *MockNetworkPlugin
+}
+
+func NewMockNetworkPlugin(ctrl *gomock.Controller) *MockNetworkPlugin {
+	mock := &MockNetworkPlugin{ctrl: ctrl}
+	mock.recorder = &_MockNetworkPluginRecorder{mock}
+	return mock
+}
+
+func (_m *MockNetworkPlugin) EXPECT() *_MockNetworkPluginRecorder {
+	return _m.recorder
+}
+
+func (_m *MockNetworkPlugin) Capabilities() sets.Int {
+	ret := _m.ctrl.Call(_m, "Capabilities")
+	ret0, _ := ret[0].(sets.Int)
+	return ret0
+}
+
+func (_m *MockNetworkPlugin) Finish() {
+	_m.ctrl.Finish()
+}
+
+func (_mr *_MockNetworkPluginRecorder) Capabilities() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Capabilities")
+}
+
+func (_m *MockNetworkPlugin) Event(_param0 string, _param1 map[string]interface{}) {
+	_m.ctrl.Call(_m, "Event", _param0, _param1)
+}
+
+func (_mr *_MockNetworkPluginRecorder) Event(arg0, arg1 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Event", arg0, arg1)
+}
+
+func (_m *MockNetworkPlugin) GetPodNetworkStatus(_param0 string, _param1 string, _param2 container.ContainerID) (*network.PodNetworkStatus, error) {
+	ret := _m.ctrl.Call(_m, "GetPodNetworkStatus", _param0, _param1, _param2)
+	ret0, _ := ret[0].(*network.PodNetworkStatus)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+func (_mr *_MockNetworkPluginRecorder) GetPodNetworkStatus(arg0, arg1, arg2 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "GetPodNetworkStatus", arg0, arg1, arg2)
+}
+
+func (_m *MockNetworkPlugin) Init(_param0 network.Host, _param1 kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	ret := _m.ctrl.Call(_m, "Init", _param0, _param1)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+func (_mr *_MockNetworkPluginRecorder) Init(arg0, arg1 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Init", arg0, arg1)
+}
+
+func (_m *MockNetworkPlugin) Name() string {
+	ret := _m.ctrl.Call(_m, "Name")
+	ret0, _ := ret[0].(string)
+	return ret0
+}
+
+func (_mr *_MockNetworkPluginRecorder) Name() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Name")
+}
+
+func (_m *MockNetworkPlugin) SetUpPod(_param0 string, _param1 string, _param2 container.ContainerID, annotations map[string]string) error {
+	ret := _m.ctrl.Call(_m, "SetUpPod", _param0, _param1, _param2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+func (_mr *_MockNetworkPluginRecorder) SetUpPod(arg0, arg1, arg2 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "SetUpPod", arg0, arg1, arg2)
+}
+
+func (_m *MockNetworkPlugin) Status() error {
+	ret := _m.ctrl.Call(_m, "Status")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+func (_mr *_MockNetworkPluginRecorder) Status() *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "Status")
+}
+
+func (_m *MockNetworkPlugin) TearDownPod(_param0 string, _param1 string, _param2 container.ContainerID) error {
+	ret := _m.ctrl.Call(_m, "TearDownPod", _param0, _param1, _param2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+func (_mr *_MockNetworkPluginRecorder) TearDownPod(arg0, arg1, arg2 interface{}) *gomock.Call {
+	return _mr.mock.ctrl.RecordCall(_mr.mock, "TearDownPod", arg0, arg1, arg2)
+}

vendor/k8s.io/kubernetes/pkg/kubelet/network/testing/plugins_test.go

@@ -0,0 +1,249 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testing
+
+import (
+	"fmt"
+	"net"
+	"sync"
+	"testing"
+
+	utilsets "k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
+	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+	"k8s.io/kubernetes/pkg/kubelet/network"
+	sysctltest "k8s.io/kubernetes/pkg/util/sysctl/testing"
+
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSelectDefaultPlugin(t *testing.T) {
+	all_plugins := []network.NetworkPlugin{}
+	plug, err := network.InitNetworkPlugin(all_plugins, "", NewFakeHost(nil), kubeletconfig.HairpinNone, "10.0.0.0/8", network.UseDefaultMTU)
+	if err != nil {
+		t.Fatalf("Unexpected error in selecting default plugin: %v", err)
+	}
+	if plug == nil {
+		t.Fatalf("Failed to select the default plugin.")
+	}
+	if plug.Name() != network.DefaultPluginName {
+		t.Errorf("Failed to select the default plugin. Expected %s. Got %s", network.DefaultPluginName, plug.Name())
+	}
+}
+
+func TestInit(t *testing.T) {
+	tests := []struct {
+		setting     string
+		expectedLen int
+	}{
+		{
+			setting:     "net/bridge/bridge-nf-call-iptables",
+			expectedLen: 1,
+		},
+		{
+			setting:     "net/bridge/bridge-nf-call-ip6tables",
+			expectedLen: 2,
+		},
+	}
+	for _, tt := range tests {
+		sysctl := sysctltest.NewFake()
+		sysctl.Settings[tt.setting] = 0
+		plug := &network.NoopNetworkPlugin{}
+		plug.Sysctl = sysctl
+		plug.Init(NewFakeHost(nil), kubeletconfig.HairpinNone, "10.0.0.0/8", network.UseDefaultMTU)
+		// Verify the sysctl specified is set
+		assert.Equal(t, 1, sysctl.Settings[tt.setting], tt.setting+" sysctl should have been set")
+		// Verify iptables is always set
+		assert.Equal(t, 1, sysctl.Settings["net/bridge/bridge-nf-call-iptables"], "net/bridge/bridge-nf-call-iptables sysctl should have been set")
+		// Verify ip6tables is only set if it existed
+		assert.Len(t, sysctl.Settings, tt.expectedLen, "length wrong for "+tt.setting)
+	}
+}
+
+func TestPluginManager(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	fnp := NewMockNetworkPlugin(ctrl)
+	defer fnp.Finish()
+	pm := network.NewPluginManager(fnp)
+
+	fnp.EXPECT().Name().Return("someNetworkPlugin").AnyTimes()
+
+	allCreatedWg := sync.WaitGroup{}
+	allCreatedWg.Add(1)
+	allDoneWg := sync.WaitGroup{}
+
+	// 10 pods, 4 setup/status/teardown runs each.  Ensure that network locking
+	// works and the pod map isn't concurrently accessed
+	for i := 0; i < 10; i++ {
+		podName := fmt.Sprintf("pod%d", i)
+		containerID := kubecontainer.ContainerID{ID: podName}
+
+		fnp.EXPECT().SetUpPod("", podName, containerID).Return(nil).Times(4)
+		fnp.EXPECT().GetPodNetworkStatus("", podName, containerID).Return(&network.PodNetworkStatus{IP: net.ParseIP("1.2.3.4")}, nil).Times(4)
+		fnp.EXPECT().TearDownPod("", podName, containerID).Return(nil).Times(4)
+
+		for x := 0; x < 4; x++ {
+			allDoneWg.Add(1)
+			go func(name string, id kubecontainer.ContainerID, num int) {
+				defer allDoneWg.Done()
+
+				// Block all goroutines from running until all have
+				// been created and are ready.  This ensures we
+				// have more pod network operations running
+				// concurrently.
+				allCreatedWg.Wait()
+
+				if err := pm.SetUpPod("", name, id, nil); err != nil {
+					t.Errorf("Failed to set up pod %q: %v", name, err)
+					return
+				}
+
+				if _, err := pm.GetPodNetworkStatus("", name, id); err != nil {
+					t.Errorf("Failed to inspect pod %q: %v", name, err)
+					return
+				}
+
+				if err := pm.TearDownPod("", name, id); err != nil {
+					t.Errorf("Failed to tear down pod %q: %v", name, err)
+					return
+				}
+			}(podName, containerID, x)
+		}
+	}
+	// Block all goroutines from running until all have been created and started
+	allCreatedWg.Done()
+
+	// Wait for them all to finish
+	allDoneWg.Wait()
+}
+
+type hookableFakeNetworkPluginSetupHook func(namespace, name string, id kubecontainer.ContainerID)
+
+type hookableFakeNetworkPlugin struct {
+	setupHook hookableFakeNetworkPluginSetupHook
+}
+
+func newHookableFakeNetworkPlugin(setupHook hookableFakeNetworkPluginSetupHook) *hookableFakeNetworkPlugin {
+	return &hookableFakeNetworkPlugin{
+		setupHook: setupHook,
+	}
+}
+
+func (p *hookableFakeNetworkPlugin) Init(host network.Host, hairpinMode kubeletconfig.HairpinMode, nonMasqueradeCIDR string, mtu int) error {
+	return nil
+}
+
+func (p *hookableFakeNetworkPlugin) Event(name string, details map[string]interface{}) {
+}
+
+func (p *hookableFakeNetworkPlugin) Name() string {
+	return "fakeplugin"
+}
+
+func (p *hookableFakeNetworkPlugin) Capabilities() utilsets.Int {
+	return utilsets.NewInt()
+}
+
+func (p *hookableFakeNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
+	if p.setupHook != nil {
+		p.setupHook(namespace, name, id)
+	}
+	return nil
+}
+
+func (p *hookableFakeNetworkPlugin) TearDownPod(string, string, kubecontainer.ContainerID) error {
+	return nil
+}
+
+func (p *hookableFakeNetworkPlugin) GetPodNetworkStatus(string, string, kubecontainer.ContainerID) (*network.PodNetworkStatus, error) {
+	return &network.PodNetworkStatus{IP: net.ParseIP("10.1.2.3")}, nil
+}
+
+func (p *hookableFakeNetworkPlugin) Status() error {
+	return nil
+}
+
+// Ensure that one pod's network operations don't block another's.  If the
+// test is successful (eg, first pod doesn't block on second) the test
+// will complete.  If unsuccessful, it will hang and get killed.
+func TestMultiPodParallelNetworkOps(t *testing.T) {
+	podWg := sync.WaitGroup{}
+	podWg.Add(1)
+
+	// Can't do this with MockNetworkPlugin because the gomock controller
+	// has its own locks which don't allow the parallel network operation
+	// to proceed.
+	didWait := false
+	fakePlugin := newHookableFakeNetworkPlugin(func(podNamespace, podName string, id kubecontainer.ContainerID) {
+		if podName == "waiter" {
+			podWg.Wait()
+			didWait = true
+		}
+	})
+	pm := network.NewPluginManager(fakePlugin)
+
+	opsWg := sync.WaitGroup{}
+
+	// Start the pod that will wait for the other to complete
+	opsWg.Add(1)
+	go func() {
+		defer opsWg.Done()
+
+		podName := "waiter"
+		containerID := kubecontainer.ContainerID{ID: podName}
+
+		// Setup will block on the runner pod completing.  If network
+		// operations locking isn't correct (eg pod network operations
+		// block other pods) setUpPod() will never return.
+		if err := pm.SetUpPod("", podName, containerID, nil); err != nil {
+			t.Errorf("Failed to set up waiter pod: %v", err)
+			return
+		}
+
+		if err := pm.TearDownPod("", podName, containerID); err != nil {
+			t.Errorf("Failed to tear down waiter pod: %v", err)
+			return
+		}
+	}()
+
+	opsWg.Add(1)
+	go func() {
+		defer opsWg.Done()
+		// Let other pod proceed
+		defer podWg.Done()
+
+		podName := "runner"
+		containerID := kubecontainer.ContainerID{ID: podName}
+
+		if err := pm.SetUpPod("", podName, containerID, nil); err != nil {
+			t.Errorf("Failed to set up runner pod: %v", err)
+			return
+		}
+
+		if err := pm.TearDownPod("", podName, containerID); err != nil {
+			t.Errorf("Failed to tear down runner pod: %v", err)
+			return
+		}
+	}()
+
+	opsWg.Wait()
+
+	if !didWait {
+		t.Errorf("waiter pod didn't wait for runner pod!")
+	}
+}