vendor update for CSI 0.3.0

vendor/k8s.io/kubernetes/pkg/util/BUILD

@@ -45,6 +45,7 @@ filegroup(
         "//pkg/util/nsenter:all-srcs",
         "//pkg/util/oom:all-srcs",
         "//pkg/util/parsers:all-srcs",
+        "//pkg/util/pod:all-srcs",
         "//pkg/util/pointer:all-srcs",
         "//pkg/util/procfs:all-srcs",
         "//pkg/util/reflector/prometheus:all-srcs",

vendor/k8s.io/kubernetes/pkg/util/configz/configz.go

@@ -19,7 +19,6 @@ package configz
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"sync"
 )
@@ -74,7 +73,7 @@ func handle(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func write(w io.Writer) error {
+func write(w http.ResponseWriter) error {
 	var b []byte
 	var err error
 	func() {
@@ -85,6 +84,7 @@ func write(w io.Writer) error {
 	if err != nil {
 		return fmt.Errorf("error marshaling json: %v", err)
 	}
+	w.Header().Set("Content-Type", "application/json")
 	_, err = w.Write(b)
 	return err
 }

vendor/k8s.io/kubernetes/pkg/util/configz/configz_test.go

@@ -44,7 +44,7 @@ func TestConfigz(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if string(body) != `{"testing":"blah"}` {
-		t.Fatalf("unexpected output: %v", err)
+		t.Fatalf("unexpected output: %s", body)
 	}
 
 	v.Set("bing")
@@ -58,7 +58,7 @@ func TestConfigz(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if string(body) != `{"testing":"bing"}` {
-		t.Fatalf("unexpected output: %v", err)
+		t.Fatalf("unexpected output: %s", body)
 	}
 
 	Delete("testing")
@@ -72,6 +72,9 @@ func TestConfigz(t *testing.T) {
 		t.Fatalf("err: %v", err)
 	}
 	if string(body) != `{}` {
-		t.Fatalf("unexpected output: %v", err)
+		t.Fatalf("unexpected output: %s", body)
+	}
+	if resp.Header.Get("Content-Type") != "application/json" {
+		t.Fatalf("unexpected Content-Type: %s", resp.Header.Get("Content-Type"))
 	}
 }

vendor/k8s.io/kubernetes/pkg/util/filesystem/defaultfs.go

@@ -72,6 +72,11 @@ func (DefaultFs) ReadFile(filename string) ([]byte, error) {
 	return ioutil.ReadFile(filename)
 }
 
+// TempDir via ioutil.TempDir
+func (DefaultFs) TempDir(dir, prefix string) (string, error) {
+	return ioutil.TempDir(dir, prefix)
+}
+
 // TempFile via ioutil.TempFile
 func (DefaultFs) TempFile(dir, prefix string) (File, error) {
 	file, err := ioutil.TempFile(dir, prefix)

vendor/k8s.io/kubernetes/pkg/util/filesystem/fakefs.go

@@ -68,6 +68,11 @@ func (fs *fakeFs) ReadFile(filename string) ([]byte, error) {
 	return fs.a.ReadFile(filename)
 }
 
+// TempDir via afero.TempDir
+func (fs *fakeFs) TempDir(dir, prefix string) (string, error) {
+	return fs.a.TempDir(dir, prefix)
+}
+
 // TempFile via afero.TempFile
 func (fs *fakeFs) TempFile(dir, prefix string) (File, error) {
 	file, err := fs.a.TempFile(dir, prefix)

vendor/k8s.io/kubernetes/pkg/util/filesystem/filesystem.go

@@ -35,6 +35,7 @@ type Filesystem interface {
 
 	// from "io/ioutil"
 	ReadFile(filename string) ([]byte, error)
+	TempDir(dir, prefix string) (string, error)
 	TempFile(dir, prefix string) (File, error)
 	ReadDir(dirname string) ([]os.FileInfo, error)
 	Walk(root string, walkFn filepath.WalkFunc) error

vendor/k8s.io/kubernetes/pkg/util/initsystem/initsystem.go

@@ -29,6 +29,9 @@ type InitSystem interface {
 	// ServiceStop tries to stop a specific service
 	ServiceStop(service string) error
 
+	// ServiceRestart tries to reload the environment and restart the specific service
+	ServiceRestart(service string) error
+
 	// ServiceExists ensures the service is defined for this init system.
 	ServiceExists(service string) bool
 
@@ -41,16 +44,34 @@ type InitSystem interface {
 
 type SystemdInitSystem struct{}
 
+func (sysd SystemdInitSystem) reloadSystemd() error {
+	if err := exec.Command("systemctl", "daemon-reload").Run(); err != nil {
+		return fmt.Errorf("failed to reload systemd: %v", err)
+	}
+	return nil
+}
+
 func (sysd SystemdInitSystem) ServiceStart(service string) error {
+	// Before we try to start any service, make sure that systemd is ready
+	if err := sysd.reloadSystemd(); err != nil {
+		return err
+	}
 	args := []string{"start", service}
-	err := exec.Command("systemctl", args...).Run()
-	return err
+	return exec.Command("systemctl", args...).Run()
+}
+
+func (sysd SystemdInitSystem) ServiceRestart(service string) error {
+	// Before we try to restart any service, make sure that systemd is ready
+	if err := sysd.reloadSystemd(); err != nil {
+		return err
+	}
+	args := []string{"restart", service}
+	return exec.Command("systemctl", args...).Run()
 }
 
 func (sysd SystemdInitSystem) ServiceStop(service string) error {
 	args := []string{"stop", service}
-	err := exec.Command("systemctl", args...).Run()
-	return err
+	return exec.Command("systemctl", args...).Run()
 }
 
 func (sysd SystemdInitSystem) ServiceExists(service string) bool {
@@ -95,6 +116,16 @@ func (sysd WindowsInitSystem) ServiceStart(service string) error {
 	return err
 }
 
+func (sysd WindowsInitSystem) ServiceRestart(service string) error {
+	if err := sysd.ServiceStop(service); err != nil {
+		return fmt.Errorf("couldn't stop service: %v", err)
+	}
+	if err := sysd.ServiceStart(service); err != nil {
+		return fmt.Errorf("couldn't start service: %v", err)
+	}
+	return nil
+}
+
 func (sysd WindowsInitSystem) ServiceStop(service string) error {
 	args := []string{"Stop-Service", service}
 	err := exec.Command("powershell", args...).Run()

vendor/k8s.io/kubernetes/pkg/util/io/writer.go

@@ -50,15 +50,24 @@ func (writer *StdWriter) WriteFile(filename string, data []byte, perm os.FileMod
 // it will not see the mounted device in its own namespace. To work around this
 // limitation one has to first enter hosts namespace (by using 'nsenter') and
 // only then write data.
-type NsenterWriter struct{}
+type NsenterWriter struct {
+	ne *nsenter.Nsenter
+}
+
+// NewNsenterWriter creates a new Writer that allows writing data to file using
+// nsenter command.
+func NewNsenterWriter(ne *nsenter.Nsenter) *NsenterWriter {
+	return &NsenterWriter{
+		ne: ne,
+	}
+}
 
 // WriteFile calls 'nsenter cat - > <the file>' and 'nsenter chmod' to create a
 // file on the host.
 func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.FileMode) error {
-	ne := nsenter.NewNsenter()
 	echoArgs := []string{"-c", fmt.Sprintf("cat > %s", filename)}
 	glog.V(5).Infof("nsenter: write data to file %s by nsenter", filename)
-	command := ne.Exec("sh", echoArgs)
+	command := writer.ne.Exec("sh", echoArgs)
 	command.SetStdin(bytes.NewBuffer(data))
 	outputBytes, err := command.CombinedOutput()
 	if err != nil {
@@ -68,7 +77,7 @@ func (writer *NsenterWriter) WriteFile(filename string, data []byte, perm os.Fil
 
 	chmodArgs := []string{fmt.Sprintf("%o", perm), filename}
 	glog.V(5).Infof("nsenter: change permissions of file %s to %s", filename, chmodArgs[0])
-	outputBytes, err = ne.Exec("chmod", chmodArgs).CombinedOutput()
+	outputBytes, err = writer.ne.Exec("chmod", chmodArgs).CombinedOutput()
 	if err != nil {
 		glog.Errorf("Output from chmod command: %v", string(outputBytes))
 		return err

vendor/k8s.io/kubernetes/pkg/util/ipset/ipset.go

@@ -87,7 +87,8 @@ type IPSet struct {
 	MaxElem int
 	// PortRange specifies the port range of bitmap:port type ipset.
 	PortRange string
-	// TODO: add comment message for ipset
+	// comment message for ipset
+	Comment string
 }
 
 // Validate checks if a given ipset is valid or not.

vendor/k8s.io/kubernetes/pkg/util/ipset/ipset_test.go

@@ -225,107 +225,215 @@ func TestCreateSet(t *testing.T) {
 	}
 }
 
+var testCases = []struct {
+	entry                *Entry
+	set                  *IPSet
+	addCombinedOutputLog [][]string
+	delCombinedOutputLog []string
+}{
+	{ // case 0
+		entry: &Entry{
+			IP:       "192.168.1.1",
+			Port:     53,
+			Protocol: ProtocolUDP,
+			SetType:  HashIPPort,
+		},
+		set: &IPSet{
+			Name: "ZERO",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "ZERO", "192.168.1.1,udp:53"},
+			{"ipset", "add", "ZERO", "192.168.1.1,udp:53", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "ZERO", "192.168.1.1,udp:53"},
+	},
+	{ // case 1
+		entry: &Entry{
+			IP:       "192.168.1.2",
+			Port:     80,
+			Protocol: ProtocolTCP,
+			SetType:  HashIPPort,
+		},
+		set: &IPSet{
+			Name: "UN",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "UN", "192.168.1.2,tcp:80"},
+			{"ipset", "add", "UN", "192.168.1.2,tcp:80", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "UN", "192.168.1.2,tcp:80"},
+	},
+	{ // case 2
+		entry: &Entry{
+			IP:       "192.168.1.3",
+			Port:     53,
+			Protocol: ProtocolUDP,
+			SetType:  HashIPPortIP,
+			IP2:      "10.20.30.1",
+		},
+		set: &IPSet{
+			Name: "DEUX",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "DEUX", "192.168.1.3,udp:53,10.20.30.1"},
+			{"ipset", "add", "DEUX", "192.168.1.3,udp:53,10.20.30.1", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "DEUX", "192.168.1.3,udp:53,10.20.30.1"},
+	},
+	{ // case 3
+		entry: &Entry{
+			IP:       "192.168.1.4",
+			Port:     80,
+			Protocol: ProtocolTCP,
+			SetType:  HashIPPortIP,
+			IP2:      "10.20.30.2",
+		},
+		set: &IPSet{
+			Name: "TROIS",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "TROIS", "192.168.1.4,tcp:80,10.20.30.2"},
+			{"ipset", "add", "TROIS", "192.168.1.4,tcp:80,10.20.30.2", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "TROIS", "192.168.1.4,tcp:80,10.20.30.2"},
+	},
+	{ // case 4
+		entry: &Entry{
+			IP:       "192.168.1.5",
+			Port:     53,
+			Protocol: ProtocolUDP,
+			SetType:  HashIPPortNet,
+			Net:      "10.20.30.0/24",
+		},
+		set: &IPSet{
+			Name: "QUATRE",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "QUATRE", "192.168.1.5,udp:53,10.20.30.0/24"},
+			{"ipset", "add", "QUATRE", "192.168.1.5,udp:53,10.20.30.0/24", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "QUATRE", "192.168.1.5,udp:53,10.20.30.0/24"},
+	},
+	{ // case 5
+		entry: &Entry{
+			IP:       "192.168.1.6",
+			Port:     80,
+			Protocol: ProtocolTCP,
+			SetType:  HashIPPortNet,
+			Net:      "10.20.40.0/24",
+		},
+		set: &IPSet{
+			Name: "CINQ",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "CINQ", "192.168.1.6,tcp:80,10.20.40.0/24"},
+			{"ipset", "add", "CINQ", "192.168.1.6,tcp:80,10.20.40.0/24", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "CINQ", "192.168.1.6,tcp:80,10.20.40.0/24"},
+	},
+	{ // case 6
+		entry: &Entry{
+			Port:     80,
+			Protocol: ProtocolTCP,
+			SetType:  BitmapPort,
+		},
+		set: &IPSet{
+			Name: "SIX",
+		},
+		addCombinedOutputLog: [][]string{
+			{"ipset", "add", "SIX", "80"},
+			{"ipset", "add", "SIX", "80", "-exist"},
+		},
+		delCombinedOutputLog: []string{"ipset", "del", "SIX", "80"},
+	},
+}
+
 func TestAddEntry(t *testing.T) {
-	testEntry := &Entry{
-		IP:       "192.168.1.1",
-		Port:     53,
-		Protocol: ProtocolUDP,
-		SetType:  HashIPPort,
-	}
-
-	testSet := &IPSet{
-		Name: "FOOBAR",
-	}
-
-	fcmd := fakeexec.FakeCmd{
-		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
-			// Success
-			func() ([]byte, error) { return []byte{}, nil },
-			// Success
-			func() ([]byte, error) { return []byte{}, nil },
-			// Failure
-			func() ([]byte, error) {
-				return []byte("ipset v6.19: Set cannot be created: set with the same name already exists"), &fakeexec.FakeExitError{Status: 1}
+	for i := range testCases {
+		fcmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				// Success
+				func() ([]byte, error) { return []byte{}, nil },
+				// Success
+				func() ([]byte, error) { return []byte{}, nil },
+				// Failure
+				func() ([]byte, error) {
+					return []byte("ipset v6.19: Set cannot be created: set with the same name already exists"), &fakeexec.FakeExitError{Status: 1}
+				},
 			},
-		},
-	}
-	fexec := fakeexec.FakeExec{
-		CommandScript: []fakeexec.FakeCommandAction{
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-		},
-	}
-	runner := New(&fexec)
-	// Create with ignoreExistErr = false, expect success
-	err := runner.AddEntry(testEntry.String(), testSet, false)
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if fcmd.CombinedOutputCalls != 1 {
-		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ipset", "add", "FOOBAR", "192.168.1.1,udp:53") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
-	}
-	// Create with ignoreExistErr = true, expect success
-	err = runner.AddEntry(testEntry.String(), testSet, true)
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if fcmd.CombinedOutputCalls != 2 {
-		t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll("ipset", "add", "FOOBAR", "192.168.1.1,udp:53", "-exist") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1])
-	}
-	// Create with ignoreExistErr = false, expect failure
-	err = runner.AddEntry(testEntry.String(), testSet, false)
-	if err == nil {
-		t.Errorf("expected failure, got nil")
+		}
+		fexec := fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{
+				func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			},
+		}
+		runner := New(&fexec)
+		// Create with ignoreExistErr = false, expect success
+		err := runner.AddEntry(testCases[i].entry.String(), testCases[i].set, false)
+		if err != nil {
+			t.Errorf("expected success, got %v", err)
+		}
+		if fcmd.CombinedOutputCalls != 1 {
+			t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
+		}
+		if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll(testCases[i].addCombinedOutputLog[0]...) {
+			t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
+		}
+		// Create with ignoreExistErr = true, expect success
+		err = runner.AddEntry(testCases[i].entry.String(), testCases[i].set, true)
+		if err != nil {
+			t.Errorf("expected success, got %v", err)
+		}
+		if fcmd.CombinedOutputCalls != 2 {
+			t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
+		}
+		if !sets.NewString(fcmd.CombinedOutputLog[1]...).HasAll(testCases[i].addCombinedOutputLog[1]...) {
+			t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[1])
+		}
+		// Create with ignoreExistErr = false, expect failure
+		err = runner.AddEntry(testCases[i].entry.String(), testCases[i].set, false)
+		if err == nil {
+			t.Errorf("expected failure, got nil")
+		}
 	}
 }
 
 func TestDelEntry(t *testing.T) {
-	// TODO: Test more set type
-	testEntry := &Entry{
-		IP:       "192.168.1.1",
-		Port:     53,
-		Protocol: ProtocolUDP,
-		SetType:  HashIPPort,
-	}
-
-	fcmd := fakeexec.FakeCmd{
-		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
-			// Success
-			func() ([]byte, error) { return []byte{}, nil },
-			// Failure
-			func() ([]byte, error) {
-				return []byte("ipset v6.19: Element cannot be deleted from the set: it's not added"), &fakeexec.FakeExitError{Status: 1}
+	for i := range testCases {
+		fcmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				// Success
+				func() ([]byte, error) { return []byte{}, nil },
+				// Failure
+				func() ([]byte, error) {
+					return []byte("ipset v6.19: Element cannot be deleted from the set: it's not added"), &fakeexec.FakeExitError{Status: 1}
+				},
 			},
-		},
-	}
-	fexec := fakeexec.FakeExec{
-		CommandScript: []fakeexec.FakeCommandAction{
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
-		},
-	}
-	runner := New(&fexec)
+		}
+		fexec := fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{
+				func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			},
+		}
+		runner := New(&fexec)
 
-	err := runner.DelEntry(testEntry.String(), "FOOBAR")
-	if err != nil {
-		t.Errorf("expected success, got %v", err)
-	}
-	if fcmd.CombinedOutputCalls != 1 {
-		t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
-	}
-	if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll("ipset", "del", "FOOBAR", "192.168.1.1,udp:53") {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
-	}
-	err = runner.DelEntry(testEntry.String(), "FOOBAR")
-	if err == nil {
-		t.Errorf("expected failure, got nil")
+		err := runner.DelEntry(testCases[i].entry.String(), testCases[i].set.Name)
+		if err != nil {
+			t.Errorf("expected success, got %v", err)
+		}
+		if fcmd.CombinedOutputCalls != 1 {
+			t.Errorf("expected 1 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
+		}
+		if !sets.NewString(fcmd.CombinedOutputLog[0]...).HasAll(testCases[i].delCombinedOutputLog...) {
+			t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[0])
+		}
+		err = runner.DelEntry(testCases[i].entry.String(), testCases[i].set.Name)
+		if err == nil {
+			t.Errorf("expected failure, got nil")
+		}
 	}
 }
 

vendor/k8s.io/kubernetes/pkg/util/iptables/iptables.go

@@ -18,10 +18,12 @@ package iptables
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"regexp"
 	"strings"
 	"sync"
+	"time"
 
 	godbus "github.com/godbus/dbus"
 	"github.com/golang/glog"
@@ -124,7 +126,7 @@ const MinCheckVersion = "1.4.11"
 const WaitMinVersion = "1.4.20"
 const WaitSecondsMinVersion = "1.4.22"
 const WaitString = "-w"
-const WaitSecondsString = "-w5"
+const WaitSecondsValue = "5"
 
 const LockfilePath16x = "/run/xtables.lock"
 
@@ -413,11 +415,18 @@ func iptablesCommand(protocol Protocol) string {
 }
 
 func (runner *runner) run(op operation, args []string) ([]byte, error) {
+	return runner.runContext(nil, op, args)
+}
+
+func (runner *runner) runContext(ctx context.Context, op operation, args []string) ([]byte, error) {
 	iptablesCmd := iptablesCommand(runner.protocol)
 	fullArgs := append(runner.waitFlag, string(op))
 	fullArgs = append(fullArgs, args...)
 	glog.V(5).Infof("running iptables %s %v", string(op), args)
-	return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
+	if ctx == nil {
+		return runner.exec.Command(iptablesCmd, fullArgs...).CombinedOutput()
+	}
+	return runner.exec.CommandContext(ctx, iptablesCmd, fullArgs...).CombinedOutput()
 	// Don't log err here - callers might not think it is an error.
 }
 
@@ -426,9 +435,8 @@ func (runner *runner) run(op operation, args []string) ([]byte, error) {
 func (runner *runner) checkRule(table Table, chain Chain, args ...string) (bool, error) {
 	if runner.hasCheck {
 		return runner.checkRuleUsingCheck(makeFullArgs(table, chain, args...))
-	} else {
-		return runner.checkRuleWithoutCheck(table, chain, args...)
 	}
+	return runner.checkRuleWithoutCheck(table, chain, args...)
 }
 
 var hexnumRE = regexp.MustCompile("0x0+([0-9])")
@@ -489,7 +497,13 @@ func (runner *runner) checkRuleWithoutCheck(table Table, chain Chain, args ...st
 
 // Executes the rule check using the "-C" flag
 func (runner *runner) checkRuleUsingCheck(args []string) (bool, error) {
-	out, err := runner.run(opCheckRule, args)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	out, err := runner.runContext(ctx, opCheckRule, args)
+	if ctx.Err() == context.DeadlineExceeded {
+		return false, fmt.Errorf("timed out while checking rules")
+	}
 	if err == nil {
 		return true, nil
 	}
@@ -558,7 +572,7 @@ func getIPTablesWaitFlag(vstring string) []string {
 	if version.LessThan(minVersion) {
 		return []string{WaitString}
 	} else {
-		return []string{WaitSecondsString}
+		return []string{WaitString, WaitSecondsValue}
 	}
 }
 
@@ -594,7 +608,7 @@ func getIPTablesRestoreWaitFlag(exec utilexec.Interface, protocol Protocol) []st
 		return nil
 	}
 
-	return []string{WaitSecondsString}
+	return []string{WaitString, WaitSecondsValue}
 }
 
 // getIPTablesRestoreVersionString runs "iptables-restore --version" to get the version string

vendor/k8s.io/kubernetes/pkg/util/iptables/iptables_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"reflect"
 	"strings"
 	"testing"
 	"time"
@@ -681,21 +682,21 @@ COMMIT
 func TestIPTablesWaitFlag(t *testing.T) {
 	testCases := []struct {
 		Version string
-		Result  string
+		Result  []string
 	}{
-		{"0.55.55", ""},
-		{"1.0.55", ""},
-		{"1.4.19", ""},
-		{"1.4.20", WaitString},
-		{"1.4.21", WaitString},
-		{"1.4.22", WaitSecondsString},
-		{"1.5.0", WaitSecondsString},
-		{"2.0.0", WaitSecondsString},
+		{"0.55.55", nil},
+		{"1.0.55", nil},
+		{"1.4.19", nil},
+		{"1.4.20", []string{WaitString}},
+		{"1.4.21", []string{WaitString}},
+		{"1.4.22", []string{WaitString, WaitSecondsValue}},
+		{"1.5.0", []string{WaitString, WaitSecondsValue}},
+		{"2.0.0", []string{WaitString, WaitSecondsValue}},
 	}
 
 	for _, testCase := range testCases {
 		result := getIPTablesWaitFlag(testCase.Version)
-		if strings.Join(result, "") != testCase.Result {
+		if !reflect.DeepEqual(result, testCase.Result) {
 			t.Errorf("For %s expected %v got %v", testCase.Version, testCase.Result, result)
 		}
 	}
@@ -730,7 +731,7 @@ func TestWaitFlagUnavailable(t *testing.T) {
 	if fcmd.CombinedOutputCalls != 3 {
 		t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
 	}
-	if sets.NewString(fcmd.CombinedOutputLog[2]...).HasAny(WaitString, WaitSecondsString) {
+	if sets.NewString(fcmd.CombinedOutputLog[2]...).Has(WaitString) {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
 }
@@ -765,7 +766,7 @@ func TestWaitFlagOld(t *testing.T) {
 	if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables", WaitString) {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
-	if sets.NewString(fcmd.CombinedOutputLog[2]...).HasAny(WaitSecondsString) {
+	if sets.NewString(fcmd.CombinedOutputLog[2]...).Has(WaitSecondsValue) {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
 }
@@ -797,10 +798,7 @@ func TestWaitFlagNew(t *testing.T) {
 	if fcmd.CombinedOutputCalls != 3 {
 		t.Errorf("expected 3 CombinedOutput() calls, got %d", fcmd.CombinedOutputCalls)
 	}
-	if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables", WaitSecondsString) {
-		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
-	}
-	if sets.NewString(fcmd.CombinedOutputLog[2]...).HasAny(WaitString) {
+	if !sets.NewString(fcmd.CombinedOutputLog[2]...).HasAll("iptables", WaitString, WaitSecondsValue) {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
 }
@@ -1165,7 +1163,7 @@ func TestRestoreAllWait(t *testing.T) {
 	}
 
 	commandSet := sets.NewString(fcmd.CombinedOutputLog[2]...)
-	if !commandSet.HasAll("iptables-restore", WaitSecondsString, "--counters", "--noflush") {
+	if !commandSet.HasAll("iptables-restore", WaitString, WaitSecondsValue, "--counters", "--noflush") {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
 
@@ -1214,8 +1212,8 @@ func TestRestoreAllWaitOldIptablesRestore(t *testing.T) {
 	if !commandSet.HasAll("iptables-restore", "--counters", "--noflush") {
 		t.Errorf("wrong CombinedOutput() log, got %s", fcmd.CombinedOutputLog[2])
 	}
-	if commandSet.HasAny(WaitSecondsString) {
-		t.Errorf("wrong CombinedOutput() log (unexpected %s option), got %s", WaitSecondsString, fcmd.CombinedOutputLog[2])
+	if commandSet.HasAll(WaitString, WaitSecondsValue) {
+		t.Errorf("wrong CombinedOutput() log (unexpected %s option), got %s", WaitString, fcmd.CombinedOutputLog[2])
 	}
 
 	if fcmd.CombinedOutputCalls != 3 {

vendor/k8s.io/kubernetes/pkg/util/iptables/testing/fake.go

@@ -33,6 +33,7 @@ const (
 	Reject      = "REJECT"
 	ToDest      = "--to-destination "
 	Recent      = "recent "
+	MatchSet    = "--match-set "
 )
 
 type Rule map[string]string
@@ -112,7 +113,7 @@ func (f *FakeIPTables) GetRules(chainName string) (rules []Rule) {
 	for _, l := range strings.Split(string(f.Lines), "\n") {
 		if strings.Contains(l, fmt.Sprintf("-A %v", chainName)) {
 			newRule := Rule(map[string]string{})
-			for _, arg := range []string{Destination, Source, DPort, Protocol, Jump, ToDest, Recent} {
+			for _, arg := range []string{Destination, Source, DPort, Protocol, Jump, ToDest, Recent, MatchSet} {
 				tok := getToken(l, arg)
 				if tok != "" {
 					newRule[arg] = tok

vendor/k8s.io/kubernetes/pkg/util/ipvs/BUILD

@@ -13,6 +13,7 @@ go_test(
     ] + select({
         "@io_bazel_rules_go//go/platform:linux": [
             "ipvs_linux_test.go",
+            "kernelcheck_linux_test.go",
         ],
         "//conditions:default": [],
     }),
@@ -20,6 +21,8 @@ go_test(
     deps = select({
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+            "//vendor/k8s.io/utils/exec/testing:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -32,36 +35,47 @@ go_library(
     ] + select({
         "@io_bazel_rules_go//go/platform:android": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:darwin": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:dragonfly": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:freebsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:linux": [
             "ipvs_linux.go",
+            "kernelcheck_linux.go",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:netbsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:openbsd": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:plan9": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:solaris": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "@io_bazel_rules_go//go/platform:windows": [
             "ipvs_unsupported.go",
+            "kernelcheck_unsupported.go",
         ],
         "//conditions:default": [],
     }),
@@ -82,6 +96,7 @@ go_library(
         "@io_bazel_rules_go//go/platform:linux": [
             "//vendor/github.com/docker/libnetwork/ipvs:go_default_library",
             "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
             "//vendor/k8s.io/utils/exec:go_default_library",
         ],
         "@io_bazel_rules_go//go/platform:nacl": [

vendor/k8s.io/kubernetes/pkg/util/ipvs/ipvs.go

@@ -61,8 +61,19 @@ const (
 	FlagPersistent = 0x1
 	// FlagHashed specify IPVS service hash flag
 	FlagHashed = 0x2
+	// IPVSProxyMode is match set up cluster with ipvs proxy model
+	IPVSProxyMode = "ipvs"
 )
 
+// Sets of IPVS required kernel modules.
+var ipvsModules = []string{
+	"ip_vs",
+	"ip_vs_rr",
+	"ip_vs_wrr",
+	"ip_vs_sh",
+	"nf_conntrack_ipv4",
+}
+
 // Equal check the equality of virtual server.
 // We don't use struct == since it doesn't work because of slice.
 func (svc *VirtualServer) Equal(other *VirtualServer) bool {

vendor/k8s.io/kubernetes/pkg/util/ipvs/ipvs_linux_test.go

@@ -306,7 +306,7 @@ func Test_toRealServer(t *testing.T) {
 	for i := range Tests {
 		got, err := toRealServer(&Tests[i].ipvsDestination)
 		if err != nil {
-			t.Errorf("case %d unexpected error: %d", i, err)
+			t.Errorf("case %d unexpected error: %v", i, err)
 		}
 		if !reflect.DeepEqual(*got, Tests[i].realServer) {
 			t.Errorf("case %d Failed to translate Destination - got %#v, want %#v", i, *got, Tests[i].realServer)
@@ -349,7 +349,7 @@ func Test_toIPVSDestination(t *testing.T) {
 	for i := range Tests {
 		got, err := toIPVSDestination(&Tests[i].realServer)
 		if err != nil {
-			t.Errorf("case %d unexpected error: %d", i, err)
+			t.Errorf("case %d unexpected error: %v", i, err)
 		}
 		if !reflect.DeepEqual(*got, Tests[i].ipvsDestination) {
 			t.Errorf("case %d failed to translate Destination - got %#v, want %#v", i, *got, Tests[i].ipvsDestination)

vendor/k8s.io/kubernetes/pkg/util/ipvs/kernelcheck_linux.go

@@ -0,0 +1,94 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilsexec "k8s.io/utils/exec"
+
+	"github.com/golang/glog"
+)
+
+// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules.
+type RequiredIPVSKernelModulesAvailableCheck struct {
+	Executor utilsexec.Interface
+}
+
+// Name returns label for RequiredIPVSKernelModulesAvailableCheck
+func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
+	return "RequiredIPVSKernelModulesAvailable"
+}
+
+// Check try to validates IPVS required kernel modules exists or not.
+// The name of function can not be changed.
+func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
+	glog.V(1).Infoln("validating the kernel module IPVS required exists in machine or not")
+
+	// Find out loaded kernel modules
+	out, err := r.Executor.Command("cut", "-f1", "-d", " ", "/proc/modules").CombinedOutput()
+	if err != nil {
+		errors = append(errors, fmt.Errorf("error getting installed ipvs required kernel modules: %v(%s)", err, out))
+		return nil, errors
+	}
+	mods := strings.Split(string(out), "\n")
+
+	wantModules := sets.NewString()
+	loadModules := sets.NewString()
+	wantModules.Insert(ipvsModules...)
+	loadModules.Insert(mods...)
+	modules := wantModules.Difference(loadModules).UnsortedList()
+
+	// Check builtin modules exist or not
+	if len(modules) != 0 {
+		kernelVersionFile := "/proc/sys/kernel/osrelease"
+		b, err := r.Executor.Command("cut", "-f1", "-d", " ", kernelVersionFile).CombinedOutput()
+		if err != nil {
+			errors = append(errors, fmt.Errorf("error getting os release kernel version: %v(%s)", err, out))
+			return nil, errors
+		}
+
+		kernelVersion := strings.TrimSpace(string(b))
+		builtinModsFilePath := fmt.Sprintf("/lib/modules/%s/modules.builtin", kernelVersion)
+		out, err := r.Executor.Command("cut", "-f1", "-d", " ", builtinModsFilePath).CombinedOutput()
+		if err != nil {
+			errors = append(errors, fmt.Errorf("error getting required builtin kernel modules: %v(%s)", err, out))
+			return nil, errors
+		}
+
+		builtInModules := sets.NewString()
+		for _, builtInMode := range ipvsModules {
+			match, _ := regexp.Match(builtInMode+".ko", out)
+			if !match {
+				builtInModules.Insert(string(builtInMode))
+			}
+		}
+		if len(builtInModules) != 0 {
+			warnings = append(warnings, fmt.Errorf(
+				"the IPVS proxier will not be used, because the following required kernel modules are not loaded: %v or no builtin kernel ipvs support: %v\n"+
+					"you can solve this problem with following methods:\n 1. Run 'modprobe -- ' to load missing kernel modules;\n"+
+					"2. Provide the missing builtin kernel ipvs support\n", modules, builtInModules))
+		}
+	}
+
+	return warnings, errors
+}

vendor/k8s.io/kubernetes/pkg/util/ipvs/kernelcheck_linux_test.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	"testing"
+
+	utilsexec "k8s.io/utils/exec"
+	fakeexec "k8s.io/utils/exec/testing"
+)
+
+func TestRequiredIPVSKernelModulesAvailableCheck(t *testing.T) {
+	cases := []struct {
+		caseName string
+
+		loadedKernel  string
+		kernelVersion string
+		builtinKernel string
+
+		expectErrors   bool
+		expectWarnings bool
+	}{
+		{
+			caseName:       "no installed kernel modules and no builtin kernel modules",
+			loadedKernel:   "",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName:      "no installed kernel modules and missing builtin kernel modules",
+			loadedKernel:  "",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName:      "no installed kernel modules and own all builtin kernel modules",
+			loadedKernel:  "",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+		{
+			caseName:       "missing installed kernel modules and no builtin kernel modules",
+			loadedKernel:   "ip_vs",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: true,
+		},
+		{
+			caseName: "own all installed kernel modules and no builtin kernel modules",
+			loadedKernel: "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" +
+				"ip_vs_rr\n" + "ip_vs_sh",
+			kernelVersion:  "3.13.0-24-generic",
+			builtinKernel:  "",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+		{
+			caseName:      "own all installed kernel modules and all builtin kernel modules",
+			loadedKernel:  "ip_vs\n" + "ip_vs_wrr\n" + "nf_conntrack_ipv4\n" + "ip_vs_rr\n" + "ip_vs_sh",
+			kernelVersion: "3.13.0-24-generic",
+			builtinKernel: "kernel/net/netfilter/ipvs/ip_vs.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_rr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_wrr.ko\n" +
+				"kernel/net/netfilter/ipvs/ip_vs_sh.ko\n" +
+				"kernel/net/ipv4/netfilter/nf_conntrack_ipv4.ko",
+			expectErrors:   false,
+			expectWarnings: false,
+		},
+	}
+
+	for i, tc := range cases {
+		fcmd := fakeexec.FakeCmd{
+			CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
+				func() ([]byte, error) { return []byte(cases[i].loadedKernel), nil },
+				func() ([]byte, error) { return []byte(cases[i].kernelVersion), nil },
+				func() ([]byte, error) { return []byte(cases[i].builtinKernel), nil },
+			},
+		}
+
+		fexec := fakeexec.FakeExec{
+			CommandScript: []fakeexec.FakeCommandAction{
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+				func(cmd string, args ...string) utilsexec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
+			},
+		}
+
+		check := RequiredIPVSKernelModulesAvailableCheck{
+			Executor: &fexec,
+		}
+		warnings, errors := check.Check()
+
+		switch {
+		case warnings != nil && !tc.expectWarnings:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected warnings for installed kernel modules %v and builtin kernel modules %v. Warnings: %v", tc.loadedKernel, tc.builtinKernel, warnings)
+		case warnings == nil && tc.expectWarnings:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected warnings for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
+		case errors != nil && !tc.expectErrors:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: unexpected errors for installed kernel modules %v and builtin kernel modules %v. errors: %v", tc.loadedKernel, tc.builtinKernel, errors)
+		case errors == nil && tc.expectErrors:
+			t.Errorf("RequiredIPVSKernelModulesAvailableCheck: expected errors for installed kernel modules %v and builtin kernel modules %v but got nothing", tc.loadedKernel, tc.builtinKernel)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/util/ipvs/kernelcheck_unsupported.go

@@ -0,0 +1,39 @@
+// +build !linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ipvs
+
+import (
+	utilsexec "k8s.io/utils/exec"
+)
+
+// RequiredIPVSKernelModulesAvailableCheck tests IPVS required kernel modules.
+type RequiredIPVSKernelModulesAvailableCheck struct {
+	Executor utilsexec.Interface
+}
+
+// Name returns label for RequiredIPVSKernelModulesAvailableCheck
+func (r RequiredIPVSKernelModulesAvailableCheck) Name() string {
+	return "RequiredIPVSKernelModulesAvailable"
+}
+
+// Check try to validates IPVS required kernel modules exists or not.
+func (r RequiredIPVSKernelModulesAvailableCheck) Check() (warnings, errors []error) {
+
+	return nil, nil
+}

vendor/k8s.io/kubernetes/pkg/util/mount/BUILD

@@ -1,10 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -72,16 +66,49 @@ go_library(
         "//conditions:default": [],
     }),
     importpath = "k8s.io/kubernetes/pkg/util/mount",
+    visibility = ["//visibility:public"],
     deps = [
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
         "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/file:go_default_library",
             "//pkg/util/io:go_default_library",
             "//pkg/util/nsenter:go_default_library",
             "//vendor/golang.org/x/sys/unix:go_default_library",
             "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//pkg/util/nsenter:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//pkg/util/file:go_default_library",
+            "//pkg/util/nsenter:go_default_library",
+        ],
         "//conditions:default": [],
     }),
 )
@@ -104,7 +131,18 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//vendor/k8s.io/utils/exec/testing:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/util/nsenter:go_default_library",
+            "//vendor/github.com/golang/glog:go_default_library",
+            "//vendor/golang.org/x/sys/unix:go_default_library",
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
+            "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(
@@ -118,4 +156,5 @@ filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

vendor/k8s.io/kubernetes/pkg/util/mount/OWNERS

@@ -3,6 +3,7 @@ reviewers:
   - saad-ali
   - jsafrane
   - msau42
+  - andyzhangx
 approvers:
   - jingxu97
   - saad-ali

vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/golang/glog"
 )
@@ -135,6 +136,34 @@ func (m *execMounter) MakeDir(pathname string) error {
 	return m.wrappedMounter.MakeDir(pathname)
 }
 
-func (m *execMounter) ExistsPath(pathname string) bool {
+func (m *execMounter) ExistsPath(pathname string) (bool, error) {
 	return m.wrappedMounter.ExistsPath(pathname)
 }
+
+func (m *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return m.wrappedMounter.PrepareSafeSubpath(subPath)
+}
+
+func (m *execMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return m.wrappedMounter.CleanSubPaths(podDir, volumeName)
+}
+
+func (m *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return m.wrappedMounter.SafeMakeDir(pathname, base, perm)
+}
+
+func (m *execMounter) GetMountRefs(pathname string) ([]string, error) {
+	return m.wrappedMounter.GetMountRefs(pathname)
+}
+
+func (m *execMounter) GetFSGroup(pathname string) (int64, error) {
+	return m.wrappedMounter.GetFSGroup(pathname)
+}
+
+func (m *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return m.wrappedMounter.GetSELinuxSupport(pathname)
+}
+
+func (m *execMounter) GetMode(pathname string) (os.FileMode, error) {
+	return m.wrappedMounter.GetMode(pathname)
+}

vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount_test.go

@@ -19,7 +19,9 @@ limitations under the License.
 package mount
 
 import (
+	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
@@ -65,7 +67,7 @@ func TestBindMount(t *testing.T) {
 			expectedArgs = []string{"-t", fsType, "-o", "bind", sourcePath, destinationPath}
 		case 2:
 			// mount -t fstype -o "remount,opts" source target
-			expectedArgs = []string{"-t", fsType, "-o", "remount," + strings.Join(mountOptions, ","), sourcePath, destinationPath}
+			expectedArgs = []string{"-t", fsType, "-o", "bind,remount," + strings.Join(mountOptions, ","), sourcePath, destinationPath}
 		}
 		if !reflect.DeepEqual(expectedArgs, args) {
 			t.Errorf("expected arguments %q, got %q", strings.Join(expectedArgs, " "), strings.Join(args, " "))
@@ -145,9 +147,36 @@ func (fm *fakeMounter) MakeFile(pathname string) error {
 func (fm *fakeMounter) MakeDir(pathname string) error {
 	return nil
 }
-func (fm *fakeMounter) ExistsPath(pathname string) bool {
-	return false
+func (fm *fakeMounter) ExistsPath(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
 }
 func (fm *fakeMounter) GetFileType(pathname string) (FileType, error) {
 	return FileTypeFile, nil
 }
+func (fm *fakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (fm *fakeMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (fm *fakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}
+
+func (fm *fakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (fm *fakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
+func (fm *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (fm *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

vendor/k8s.io/kubernetes/pkg/util/mount/exec_mount_unsupported.go

@@ -20,6 +20,7 @@ package mount
 
 import (
 	"errors"
+	"os"
 )
 
 type execMounter struct{}
@@ -82,6 +83,34 @@ func (mounter *execMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *execMounter) ExistsPath(pathname string) bool {
-	return true
+func (mounter *execMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
+}
+
+func (mounter *execMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (mounter *execMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (mounter *execMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}
+
+func (mounter *execMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *execMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
+func (mounter *execMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (mounter *execMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
 }

vendor/k8s.io/kubernetes/pkg/util/mount/fake.go

@@ -17,6 +17,7 @@ limitations under the License.
 package mount
 
 import (
+	"errors"
 	"os"
 	"path/filepath"
 	"sync"
@@ -58,8 +59,10 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options
 	f.mutex.Lock()
 	defer f.mutex.Unlock()
 
-	// find 'bind' option
+	opts := []string{}
+
 	for _, option := range options {
+		// find 'bind' option
 		if option == "bind" {
 			// This is a bind-mount. In order to mimic linux behaviour, we must
 			// use the original device of the bind-mount as the real source.
@@ -78,7 +81,11 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options
 					break
 				}
 			}
-			break
+		}
+		// find 'ro' option
+		if option == "ro" {
+			// reuse MountPoint.Opts field to mark mount as readonly
+			opts = append(opts, "ro")
 		}
 	}
 
@@ -88,7 +95,7 @@ func (f *FakeMounter) Mount(source string, target string, fstype string, options
 		absTarget = target
 	}
 
-	f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype})
+	f.MountPoints = append(f.MountPoints, MountPoint{Device: source, Path: absTarget, Type: fstype, Opts: opts})
 	glog.V(5).Infof("Fake mounter: mounted %s to %s", source, absTarget)
 	f.Log = append(f.Log, FakeAction{Action: FakeActionMount, Target: absTarget, Source: source, FSType: fstype})
 	return nil
@@ -194,6 +201,38 @@ func (f *FakeMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (f *FakeMounter) ExistsPath(pathname string) bool {
-	return false
+func (f *FakeMounter) ExistsPath(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (f *FakeMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (f *FakeMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+func (mounter *FakeMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}
+
+func (f *FakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		// Ignore error in FakeMounter, because we actually didn't create files.
+		realpath = pathname
+	}
+	return getMountRefsByDev(f, realpath)
+}
+
+func (f *FakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("GetFSGroup not implemented")
+}
+
+func (f *FakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("GetSELinuxSupport not implemented")
+}
+
+func (f *FakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
 }

vendor/k8s.io/kubernetes/pkg/util/mount/mount.go

@@ -19,6 +19,7 @@ limitations under the License.
 package mount
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -83,9 +84,58 @@ type Interface interface {
 	// MakeDir creates a new directory.
 	// Will operate in the host mount namespace if kubelet is running in a container
 	MakeDir(pathname string) error
-	// ExistsPath checks whether the path exists.
-	// Will operate in the host mount namespace if kubelet is running in a container
-	ExistsPath(pathname string) bool
+	// SafeMakeDir creates subdir within given base. It makes sure that the
+	// created directory does not escape given base directory mis-using
+	// symlinks. Note that the function makes sure that it creates the directory
+	// somewhere under the base, nothing else. E.g. if the directory already
+	// exists, it may exist outside of the base due to symlinks.
+	// This method should be used if the directory to create is inside volume
+	// that's under user control. User must not be able to use symlinks to
+	// escape the volume to create directories somewhere else.
+	SafeMakeDir(subdir string, base string, perm os.FileMode) error
+	// Will operate in the host mount namespace if kubelet is running in a container.
+	// Error is returned on any other error than "file not found".
+	ExistsPath(pathname string) (bool, error)
+	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
+	// pod volume directory.
+	CleanSubPaths(podDir string, volumeName string) error
+	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
+	// that's 1) inside given volumePath and 2) immutable after this call.
+	//
+	// newHostPath - location of prepared subPath. It should be used instead of
+	// hostName when running the container.
+	// cleanupAction - action to run when the container is running or it failed to start.
+	//
+	// CleanupAction must be called immediately after the container with given
+	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
+	// when the pod finishes.
+	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+	// GetMountRefs finds all mount references to the path, returns a
+	// list of paths. Path could be a mountpoint path, device or a normal
+	// directory (for bind mount).
+	GetMountRefs(pathname string) ([]string, error)
+	// GetFSGroup returns FSGroup of the path.
+	GetFSGroup(pathname string) (int64, error)
+	// GetSELinuxSupport returns true if given path is on a mount that supports
+	// SELinux.
+	GetSELinuxSupport(pathname string) (bool, error)
+	// GetMode returns permissions of the path.
+	GetMode(pathname string) (os.FileMode, error)
+}
+
+type Subpath struct {
+	// index of the VolumeMount for this container
+	VolumeMountIndex int
+	// Full path to the subpath directory on the host
+	Path string
+	// name of the volume that is a valid directory name.
+	VolumeName string
+	// Full path to the volume path
+	VolumePath string
+	// Path to the pod's directory, including pod UID
+	PodDir string
+	// Name of the container
+	ContainerName string
 }
 
 // Exec executes command where mount utilities are. This can be either the host,
@@ -129,22 +179,19 @@ func (mounter *SafeFormatAndMount) FormatAndMount(source string, target string,
 	return mounter.formatAndMount(source, target, fstype, options)
 }
 
-// GetMountRefsByDev finds all references to the device provided
+// getMountRefsByDev finds all references to the device provided
 // by mountPath; returns a list of paths.
-func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
+// Note that mountPath should be path after the evaluation of any symblolic links.
+func getMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
 	mps, err := mounter.List()
 	if err != nil {
 		return nil, err
 	}
-	slTarget, err := filepath.EvalSymlinks(mountPath)
-	if err != nil {
-		slTarget = mountPath
-	}
 
 	// Finding the device mounted to mountPath
 	diskDev := ""
 	for i := range mps {
-		if slTarget == mps[i].Path {
+		if mountPath == mps[i].Path {
 			diskDev = mps[i].Device
 			break
 		}
@@ -153,8 +200,8 @@ func GetMountRefsByDev(mounter Interface, mountPath string) ([]string, error) {
 	// Find all references to the device.
 	var refs []string
 	for i := range mps {
-		if mps[i].Device == diskDev || mps[i].Device == slTarget {
-			if mps[i].Path != slTarget {
+		if mps[i].Device == diskDev || mps[i].Device == mountPath {
+			if mps[i].Path != mountPath {
 				refs = append(refs, mps[i].Path)
 			}
 		}
@@ -237,7 +284,13 @@ func IsNotMountPoint(mounter Interface, file string) (bool, error) {
 // The list equals:
 //   options - 'bind' + 'remount' (no duplicate)
 func isBind(options []string) (bool, []string) {
-	bindRemountOpts := []string{"remount"}
+	// Because we have an FD opened on the subpath bind mount, the "bind" option
+	// needs to be included, otherwise the mount target will error as busy if you
+	// remount as readonly.
+	//
+	// As a consequence, all read only bind mounts will no longer change the underlying
+	// volume mount to be read only.
+	bindRemountOpts := []string{"bind", "remount"}
 	bind := false
 
 	if len(options) != 0 {
@@ -274,3 +327,56 @@ func HasMountRefs(mountPath string, mountRefs []string) bool {
 	}
 	return count > 0
 }
+
+// pathWithinBase checks if give path is within given base directory.
+func pathWithinBase(fullPath, basePath string) bool {
+	rel, err := filepath.Rel(basePath, fullPath)
+	if err != nil {
+		return false
+	}
+	if startsWithBackstep(rel) {
+		// Needed to escape the base path
+		return false
+	}
+	return true
+}
+
+// startsWithBackstep checks if the given path starts with a backstep segment
+func startsWithBackstep(rel string) bool {
+	// normalize to / and check for ../
+	return rel == ".." || strings.HasPrefix(filepath.ToSlash(rel), "../")
+}
+
+// getFileType checks for file/directory/socket and block/character devices
+func getFileType(pathname string) (FileType, error) {
+	var pathType FileType
+	info, err := os.Stat(pathname)
+	if os.IsNotExist(err) {
+		return pathType, fmt.Errorf("path %q does not exist", pathname)
+	}
+	// err in call to os.Stat
+	if err != nil {
+		return pathType, err
+	}
+
+	// checks whether the mode is the target mode
+	isSpecificMode := func(mode, targetMode os.FileMode) bool {
+		return mode&targetMode == targetMode
+	}
+
+	mode := info.Mode()
+	if mode.IsDir() {
+		return FileTypeDirectory, nil
+	} else if mode.IsRegular() {
+		return FileTypeFile, nil
+	} else if isSpecificMode(mode, os.ModeSocket) {
+		return FileTypeSocket, nil
+	} else if isSpecificMode(mode, os.ModeDevice) {
+		if isSpecificMode(mode, os.ModeCharDevice) {
+			return FileTypeCharDev, nil
+		}
+		return FileTypeBlockDev, nil
+	}
+
+	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
+}

vendor/k8s.io/kubernetes/pkg/util/mount/mount_linux.go

@@ -21,6 +21,7 @@ package mount
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -32,6 +33,7 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/sys/unix"
 	"k8s.io/apimachinery/pkg/util/sets"
+	utilfile "k8s.io/kubernetes/pkg/util/file"
 	utilio "k8s.io/kubernetes/pkg/util/io"
 	utilexec "k8s.io/utils/exec"
 )
@@ -41,6 +43,8 @@ const (
 	maxListTries = 3
 	// Number of fields per line in /proc/mounts as per the fstab man page.
 	expectedNumFieldsPerLine = 6
+	// At least number of fields per line in /proc/<pid>/mountinfo.
+	expectedAtLeastNumFieldsPerMountInfo = 10
 	// Location of the mount file to use
 	procMountsPath = "/proc/mounts"
 	// Location of the mountinfo file
@@ -49,6 +53,13 @@ const (
 	fsckErrorsCorrected = 1
 	// 'fsck' found errors but exited without correcting them
 	fsckErrorsUncorrected = 4
+
+	// place for subpath mounts
+	containerSubPathDirectoryName = "volume-subpaths"
+	// syscall.Openat flags used to traverse directories not following symlinks
+	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
+	// flags for getting file descriptor without following the symlink
+	openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
 )
 
 // Mounter provides the default implementation of mount.Interface
@@ -415,31 +426,7 @@ func (mounter *Mounter) MakeRShared(path string) error {
 }
 
 func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
-	var pathType FileType
-	finfo, err := os.Stat(pathname)
-	if os.IsNotExist(err) {
-		return pathType, fmt.Errorf("path %q does not exist", pathname)
-	}
-	// err in call to os.Stat
-	if err != nil {
-		return pathType, err
-	}
-
-	mode := finfo.Sys().(*syscall.Stat_t).Mode
-	switch mode & syscall.S_IFMT {
-	case syscall.S_IFSOCK:
-		return FileTypeSocket, nil
-	case syscall.S_IFBLK:
-		return FileTypeBlockDev, nil
-	case syscall.S_IFCHR:
-		return FileTypeCharDev, nil
-	case syscall.S_IFDIR:
-		return FileTypeDirectory, nil
-	case syscall.S_IFREG:
-		return FileTypeFile, nil
-	}
-
-	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
+	return getFileType(pathname)
 }
 
 func (mounter *Mounter) MakeDir(pathname string) error {
@@ -463,12 +450,8 @@ func (mounter *Mounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *Mounter) ExistsPath(pathname string) bool {
-	_, err := os.Stat(pathname)
-	if err != nil {
-		return false
-	}
-	return true
+func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
+	return utilfile.FileExists(pathname)
 }
 
 // formatAndMount uses unix utils to format and mount the given disk
@@ -527,7 +510,11 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string,
 			}
 
 			if fstype == "ext4" || fstype == "ext3" {
-				args = []string{"-F", source}
+				args = []string{
+					"-F",  // Force flag
+					"-m0", // Zero blocks reserved for super-user
+					source,
+				}
 			}
 			glog.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args)
 			_, err := mounter.Exec.Run("mkfs."+fstype, args...)
@@ -607,27 +594,14 @@ func (mounter *SafeFormatAndMount) GetDiskFormat(disk string) (string, error) {
 
 // isShared returns true, if given path is on a mount point that has shared
 // mount propagation.
-func isShared(path string, filename string) (bool, error) {
-	infos, err := parseMountInfo(filename)
+func isShared(mount string, mountInfoPath string) (bool, error) {
+	info, err := findMountInfo(mount, mountInfoPath)
 	if err != nil {
 		return false, err
 	}
 
-	// process /proc/xxx/mountinfo in backward order and find the first mount
-	// point that is prefix of 'path' - that's the mount where path resides
-	var info *mountInfo
-	for i := len(infos) - 1; i >= 0; i-- {
-		if strings.HasPrefix(path, infos[i].mountPoint) {
-			info = &infos[i]
-			break
-		}
-	}
-	if info == nil {
-		return false, fmt.Errorf("cannot find mount point for %q", path)
-	}
-
 	// parse optional parameters
-	for _, opt := range info.optional {
+	for _, opt := range info.optionalFields {
 		if strings.HasPrefix(opt, "shared:") {
 			return true, nil
 		}
@@ -635,10 +609,28 @@ func isShared(path string, filename string) (bool, error) {
 	return false, nil
 }
 
+// This represents a single line in /proc/<pid>/mountinfo.
 type mountInfo struct {
+	// Unique ID for the mount (maybe reused after umount).
+	id int
+	// The ID of the parent mount (or of self for the root of this mount namespace's mount tree).
+	parentID int
+	// The value of `st_dev` for files on this filesystem.
+	majorMinor string
+	// The pathname of the directory in the filesystem which forms the root of this mount.
+	root string
+	// Mount source, filesystem-specific information. e.g. device, tmpfs name.
+	source string
+	// Mount point, the pathname of the mount point.
 	mountPoint string
-	// list of "optional parameters", mount propagation is one of them
-	optional []string
+	// Optional fieds, zero or more fields of the form "tag[:value]".
+	optionalFields []string
+	// The filesystem type in the form "type[.subtype]".
+	fsType string
+	// Per-mount options.
+	mountOptions []string
+	// Per-superblock options.
+	superOptions []string
 }
 
 // parseMountInfo parses /proc/xxx/mountinfo.
@@ -655,22 +647,66 @@ func parseMountInfo(filename string) ([]mountInfo, error) {
 			// the last split() item is empty string following the last \n
 			continue
 		}
+		// See `man proc` for authoritative description of format of the file.
 		fields := strings.Fields(line)
-		if len(fields) < 7 {
-			return nil, fmt.Errorf("wrong number of fields in (expected %d, got %d): %s", 8, len(fields), line)
+		if len(fields) < expectedAtLeastNumFieldsPerMountInfo {
+			return nil, fmt.Errorf("wrong number of fields in (expected at least %d, got %d): %s", expectedAtLeastNumFieldsPerMountInfo, len(fields), line)
+		}
+		id, err := strconv.Atoi(fields[0])
+		if err != nil {
+			return nil, err
+		}
+		parentID, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return nil, err
 		}
 		info := mountInfo{
-			mountPoint: fields[4],
-			optional:   []string{},
+			id:           id,
+			parentID:     parentID,
+			majorMinor:   fields[2],
+			root:         fields[3],
+			mountPoint:   fields[4],
+			mountOptions: strings.Split(fields[5], ","),
 		}
-		for i := 6; i < len(fields) && fields[i] != "-"; i++ {
-			info.optional = append(info.optional, fields[i])
+		// All fields until "-" are "optional fields".
+		i := 6
+		for ; i < len(fields) && fields[i] != "-"; i++ {
+			info.optionalFields = append(info.optionalFields, fields[i])
 		}
+		// Parse the rest 3 fields.
+		i += 1
+		if len(fields)-i < 3 {
+			return nil, fmt.Errorf("expect 3 fields in %s, got %d", line, len(fields)-i)
+		}
+		info.fsType = fields[i]
+		info.source = fields[i+1]
+		info.superOptions = strings.Split(fields[i+2], ",")
 		infos = append(infos, info)
 	}
 	return infos, nil
 }
 
+func findMountInfo(path, mountInfoPath string) (mountInfo, error) {
+	infos, err := parseMountInfo(mountInfoPath)
+	if err != nil {
+		return mountInfo{}, err
+	}
+
+	// process /proc/xxx/mountinfo in backward order and find the first mount
+	// point that is prefix of 'path' - that's the mount where path resides
+	var info *mountInfo
+	for i := len(infos) - 1; i >= 0; i-- {
+		if pathWithinBase(path, infos[i].mountPoint) {
+			info = &infos[i]
+			break
+		}
+	}
+	if info == nil {
+		return mountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
+	}
+	return *info, nil
+}
+
 // doMakeRShared is common implementation of MakeRShared on Linux. It checks if
 // path is shared and bind-mounts it as rshared if needed. mountCmd and
 // mountArgs are expected to contain mount-like command, doMakeRShared will add
@@ -698,3 +734,624 @@ func doMakeRShared(path string, mountInfoFilename string) error {
 
 	return nil
 }
+
+// getSELinuxSupport is common implementation of GetSELinuxSupport on Linux.
+func getSELinuxSupport(path string, mountInfoFilename string) (bool, error) {
+	info, err := findMountInfo(path, mountInfoFilename)
+	if err != nil {
+		return false, err
+	}
+
+	// "seclabel" can be both in mount options and super options.
+	for _, opt := range info.superOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	for _, opt := range info.mountOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	newHostPath, err = doBindSubPath(mounter, subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+func safeOpenSubPath(mounter Interface, subpath Subpath) (int, error) {
+	if !pathWithinBase(subpath.Path, subpath.VolumePath) {
+		return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
+	}
+	fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
+	if err != nil {
+		return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
+	}
+	return fd, nil
+}
+
+// prepareSubpathTarget creates target for bind-mount of subpath. It returns
+// "true" when the target already exists and something is mounted there.
+// Given Subpath must have all paths with already resolved symlinks and with
+// paths relevant to kubelet (when it runs in a container).
+// This function is called also by NsEnterMounter. It works because
+// /var/lib/kubelet is mounted from the host into the container with Kubelet as
+// /var/lib/kubelet too.
+func prepareSubpathTarget(mounter Interface, subpath Subpath) (bool, string, error) {
+	// Early check for already bind-mounted subpath.
+	bindPathTarget := getSubpathBindTarget(subpath)
+	notMount, err := IsNotMountPoint(mounter, bindPathTarget)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
+		}
+		// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
+		notMount = true
+	}
+	if !notMount {
+		// It's already mounted
+		glog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
+		return true, bindPathTarget, nil
+	}
+
+	// bindPathTarget is in /var/lib/kubelet and thus reachable without any
+	// translation even to containerized kubelet.
+	bindParent := filepath.Dir(bindPathTarget)
+	err = os.MkdirAll(bindParent, 0750)
+	if err != nil && !os.IsExist(err) {
+		return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
+	}
+
+	t, err := os.Lstat(subpath.Path)
+	if err != nil {
+		return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
+	}
+
+	if t.Mode()&os.ModeDir > 0 {
+		if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
+			return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
+		}
+	} else {
+		// "/bin/touch <bindPathTarget>".
+		// A file is enough for all possible targets (symlink, device, pipe,
+		// socket, ...), bind-mounting them into a file correctly changes type
+		// of the target file.
+		if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
+			return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
+		}
+	}
+	return false, bindPathTarget, nil
+}
+
+func getSubpathBindTarget(subpath Subpath) string {
+	// containerName is DNS label, i.e. safe as a directory name.
+	return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
+}
+
+func doBindSubPath(mounter Interface, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs on the host:
+	// - safely open the subpath
+	// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
+	// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	newPath, err := filepath.EvalSymlinks(subpath.Path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
+	subpath.VolumePath = newVolumePath
+	subpath.Path = newPath
+
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			glog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	kubeletPid := os.Getpid()
+	mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
+
+	// Do the bind mount
+	options := []string{"bind"}
+	glog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
+	if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
+	}
+	success = true
+
+	glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(mounter, podDir, volumeName)
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error {
+	// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
+	subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
+	glog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
+
+	containerDirs, err := ioutil.ReadDir(subPathDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("error reading %s: %s", subPathDir, err)
+	}
+
+	for _, containerDir := range containerDirs {
+		if !containerDir.IsDir() {
+			glog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
+			continue
+		}
+		glog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
+
+		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
+		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
+		subPaths, err := ioutil.ReadDir(fullContainerDirPath)
+		if err != nil {
+			return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err)
+		}
+		for _, subPath := range subPaths {
+			if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil {
+				return err
+			}
+		}
+		// Whole container has been processed, remove its directory.
+		if err := os.Remove(fullContainerDirPath); err != nil {
+			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
+		}
+		glog.V(5).Infof("Removed %s", fullContainerDirPath)
+	}
+	// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
+	if err := os.Remove(subPathDir); err != nil {
+		return fmt.Errorf("error deleting %s: %s", subPathDir, err)
+	}
+	glog.V(5).Infof("Removed %s", subPathDir)
+
+	// Remove entire subpath directory if it's the last one
+	podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
+	if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
+	}
+	glog.V(5).Infof("Removed %s", podSubPathDir)
+	return nil
+}
+
+// doCleanSubPath tears down the single subpath bind mount
+func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string) error {
+	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
+	glog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
+	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
+	notMnt, err := IsNotMountPoint(mounter, fullSubPath)
+	if err != nil {
+		return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err)
+	}
+	// Unmount it
+	if !notMnt {
+		if err = mounter.Unmount(fullSubPath); err != nil {
+			return fmt.Errorf("error unmounting %s: %s", fullSubPath, err)
+		}
+		glog.V(5).Infof("Unmounted %s", fullSubPath)
+	}
+	// Remove it *non*-recursively, just in case there were some hiccups.
+	if err = os.Remove(fullSubPath); err != nil {
+		return fmt.Errorf("error deleting %s: %s", fullSubPath, err)
+	}
+	glog.V(5).Infof("Removed %s", fullSubPath)
+	return nil
+}
+
+// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
+func cleanSubPath(mounter Interface, subpath Subpath) error {
+	containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
+
+	// Clean subdir bindmount
+	if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// Recusively remove directories if empty
+	if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
+// if it is empty.  It stops once it encounters a directory that has content
+func removeEmptyDirs(baseDir, endDir string) error {
+	if !pathWithinBase(endDir, baseDir) {
+		return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
+	}
+
+	for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
+		s, err := os.Stat(curDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				glog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
+				continue
+			}
+			return fmt.Errorf("error stat %q: %v", curDir, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("path %q not a directory", curDir)
+		}
+
+		err = os.Remove(curDir)
+		if os.IsExist(err) {
+			glog.V(5).Infof("Directory %q not empty, not removing", curDir)
+			break
+		} else if err != nil {
+			return fmt.Errorf("error removing directory %q: %v", curDir, err)
+		}
+		glog.V(5).Infof("Removed directory %q", curDir)
+	}
+	return nil
+}
+
+func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return searchMountPoints(realpath, procMountInfoPath)
+}
+
+func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return getSELinuxSupport(pathname, procMountInfoPath)
+}
+
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return getFSGroup(realpath)
+}
+
+func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
+	return getMode(pathname)
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+func getFSGroup(pathname string) (int64, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return int64(info.Sys().(*syscall.Stat_t).Gid), nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+func getMode(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter. Both pathname
+// and base must be either already resolved symlinks or thet will be resolved in
+// kubelet's mount namespace (in case it runs containerized).
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !pathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			glog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", existingPath, err)
+	}
+	if !pathWithinBase(fullExistingPath, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	parentFD, err := doSafeOpen(fullExistingPath, base)
+	if err != nil {
+		return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
+	}
+	childFD := -1
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+			}
+		}
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink.
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		glog.V(4).Infof("Creating %s", dir)
+		err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
+		if err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		// Dive into the created directory
+		childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
+		if err != nil {
+			return fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+		// We can be sure that childFD is safe to use. It could be changed
+		// by user after Mkdirat() and before Openat(), however:
+		// - it could not be changed to symlink - we use nofollowFlags
+		// - it could be changed to a file (or device, pipe, socket, ...)
+		//   but either subsequent Mkdirat() fails or we mount this file
+		//   to user's container. Security is no violated in both cases
+		//   and user either gets error or the file that it can already access.
+
+		if err = syscall.Close(parentFD); err != nil {
+			glog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+		}
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// Everything was created. mkdirat(..., perm) above was affected by current
+	// umask and we must apply the right permissions to the last directory
+	// (that's the one that will be available to the container as subpath)
+	// so user can read/write it. This is the behavior of previous code.
+	// TODO: chmod all created directories, not just the last one.
+	// parentFD is the last created directory.
+
+	// Translate perm (os.FileMode) to uint32 that fchmod() expects
+	kernelPerm := uint32(perm & os.ModePerm)
+	if perm&os.ModeSetgid > 0 {
+		kernelPerm |= syscall.S_ISGID
+	}
+	if perm&os.ModeSetuid > 0 {
+		kernelPerm |= syscall.S_ISUID
+	}
+	if perm&os.ModeSticky > 0 {
+		kernelPerm |= syscall.S_ISVTX
+	}
+	if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
+		return fmt.Errorf("chmod %q failed: %s", currentPath, err)
+	}
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
+	// This should be faster than looping through all dirs and calling os.Stat()
+	// on each of them, as the symlinks are resolved only once with OpenAt().
+	currentPath := base
+	fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
+	if err != nil {
+		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
+	}
+	defer func() {
+		if err = syscall.Close(fd); err != nil {
+			glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+	}()
+	for i, dir := range dirs {
+		// Using O_PATH here will prevent hangs in case user replaces directory with
+		// fifo
+		childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return currentPath, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+		if err = syscall.Close(fd); err != nil {
+			glog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+		fd = childFD
+		currentPath = filepath.Join(currentPath, dir)
+	}
+	return pathname, []string{}, nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+// Open path and return its fd.
+// Symlinks are disallowed (pathname must already resolve symlinks),
+// and the path must be within the base directory.
+func doSafeOpen(pathname string, base string) (int, error) {
+	pathname = filepath.Clean(pathname)
+	base = filepath.Clean(base)
+
+	// Calculate segments to follow
+	subpath, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return -1, err
+	}
+	segments := strings.Split(subpath, string(filepath.Separator))
+
+	// Assumption: base is the only directory that we have under control.
+	// Base dir is not allowed to be a symlink.
+	parentFD, err := syscall.Open(base, nofollowFlags, 0)
+	if err != nil {
+		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
+	}
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
+			}
+		}
+	}()
+
+	childFD := -1
+	defer func() {
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				glog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := base
+
+	// Follow the segments one by one using openat() to make
+	// sure the user cannot change already existing directories into symlinks.
+	for _, seg := range segments {
+		currentPath = filepath.Join(currentPath, seg)
+		if !pathWithinBase(currentPath, base) {
+			return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
+		}
+
+		glog.V(5).Infof("Opening path %s", currentPath)
+		childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
+		if err != nil {
+			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+
+		var deviceStat unix.Stat_t
+		err := unix.Fstat(childFD, &deviceStat)
+		if err != nil {
+			return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
+		}
+		fileFmt := deviceStat.Mode & syscall.S_IFMT
+		if fileFmt == syscall.S_IFLNK {
+			return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
+		}
+
+		// Close parentFD
+		if err = syscall.Close(parentFD); err != nil {
+			return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
+		}
+		// Set child to new parent
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// We made it to the end, return this fd, don't close it
+	finalFD := parentFD
+	parentFD = -1
+
+	return finalFD, nil
+}
+
+// searchMountPoints finds all mount references to the source, returns a list of
+// mountpoints.
+// This function assumes source cannot be device.
+// Some filesystems may share a source name, e.g. tmpfs. And for bind mounting,
+// it's possible to mount a non-root path of a filesystem, so we need to use
+// root path and major:minor to represent mount source uniquely.
+// This implementation is shared between Linux and NsEnterMounter
+func searchMountPoints(hostSource, mountInfoPath string) ([]string, error) {
+	mis, err := parseMountInfo(mountInfoPath)
+	if err != nil {
+		return nil, err
+	}
+
+	mountID := 0
+	rootPath := ""
+	majorMinor := ""
+
+	// Finding the underlying root path and major:minor if possible.
+	// We need search in backward order because it's possible for later mounts
+	// to overlap earlier mounts.
+	for i := len(mis) - 1; i >= 0; i-- {
+		if hostSource == mis[i].mountPoint || pathWithinBase(hostSource, mis[i].mountPoint) {
+			// If it's a mount point or path under a mount point.
+			mountID = mis[i].id
+			rootPath = filepath.Join(mis[i].root, strings.TrimPrefix(hostSource, mis[i].mountPoint))
+			majorMinor = mis[i].majorMinor
+			break
+		}
+	}
+
+	if rootPath == "" || majorMinor == "" {
+		return nil, fmt.Errorf("failed to get root path and major:minor for %s", hostSource)
+	}
+
+	var refs []string
+	for i := range mis {
+		if mis[i].id == mountID {
+			// Ignore mount entry for mount source itself.
+			continue
+		}
+		if mis[i].root == rootPath && mis[i].majorMinor == majorMinor {
+			refs = append(refs, mis[i].mountPoint)
+		}
+	}
+
+	return refs, nil
+}

vendor/k8s.io/kubernetes/pkg/util/mount/mount_unsupported.go

@@ -20,12 +20,15 @@ package mount
 
 import (
 	"errors"
+	"os"
 )
 
 type Mounter struct {
 	mounterPath string
 }
 
+var unsupportedErr = errors.New("util/mount on this platform is not supported")
+
 // New returns a mount.Interface for the current system.
 // It provides options to override the default mounter behavior.
 // mounterPath allows using an alternative to `/bin/mount` for mounting.
@@ -36,21 +39,21 @@ func New(mounterPath string) Interface {
 }
 
 func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error {
-	return nil
+	return unsupportedErr
 }
 
 func (mounter *Mounter) Unmount(target string) error {
-	return nil
+	return unsupportedErr
 }
 
 // GetMountRefs finds all other references to the device referenced
 // by mountPath; returns a list of paths.
 func GetMountRefs(mounter Interface, mountPath string) ([]string, error) {
-	return []string{}, nil
+	return []string{}, unsupportedErr
 }
 
 func (mounter *Mounter) List() ([]MountPoint, error) {
-	return []MountPoint{}, nil
+	return []MountPoint{}, unsupportedErr
 }
 
 func (mounter *Mounter) IsMountPointMatch(mp MountPoint, dir string) bool {
@@ -62,27 +65,27 @@ func (mounter *Mounter) IsNotMountPoint(dir string) (bool, error) {
 }
 
 func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
-	return true, nil
+	return true, unsupportedErr
 }
 
 func (mounter *Mounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
-	return "", nil
+	return "", unsupportedErr
 }
 
 func getDeviceNameFromMount(mounter Interface, mountPath, pluginDir string) (string, error) {
-	return "", nil
+	return "", unsupportedErr
 }
 
 func (mounter *Mounter) DeviceOpened(pathname string) (bool, error) {
-	return false, nil
+	return false, unsupportedErr
 }
 
 func (mounter *Mounter) PathIsDevice(pathname string) (bool, error) {
-	return true, nil
+	return true, unsupportedErr
 }
 
 func (mounter *Mounter) MakeRShared(path string) error {
-	return nil
+	return unsupportedErr
 }
 
 func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
@@ -90,21 +93,49 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string,
 }
 
 func (mounter *SafeFormatAndMount) diskLooksUnformatted(disk string) (bool, error) {
-	return true, nil
+	return true, unsupportedErr
 }
 
 func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
-	return FileType("fake"), errors.New("not implemented")
+	return FileType("fake"), unsupportedErr
 }
 
 func (mounter *Mounter) MakeDir(pathname string) error {
-	return nil
+	return unsupportedErr
 }
 
 func (mounter *Mounter) MakeFile(pathname string) error {
-	return nil
+	return unsupportedErr
 }
 
-func (mounter *Mounter) ExistsPath(pathname string) bool {
-	return true
+func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
+}
+
+func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, unsupportedErr
+}
+
+func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
+	return unsupportedErr
+}
+
+func (mounter *Mounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return unsupportedErr
+}
+
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
+func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
 }

vendor/k8s.io/kubernetes/pkg/util/mount/mount_windows.go

@@ -29,6 +29,8 @@ import (
 	"syscall"
 
 	"github.com/golang/glog"
+
+	utilfile "k8s.io/kubernetes/pkg/util/file"
 )
 
 // Mounter provides the default implementation of mount.Interface
@@ -145,7 +147,15 @@ func (mounter *Mounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	}
 	// If current file is a symlink, then it is a mountpoint.
 	if stat.Mode()&os.ModeSymlink != 0 {
-		return false, nil
+		target, err := os.Readlink(file)
+		if err != nil {
+			return true, fmt.Errorf("readlink error: %v", err)
+		}
+		exists, err := mounter.ExistsPath(target)
+		if err != nil {
+			return true, err
+		}
+		return !exists, nil
 	}
 
 	return true, nil
@@ -201,31 +211,7 @@ func (mounter *Mounter) MakeRShared(path string) error {
 
 // GetFileType checks for sockets/block/character devices
 func (mounter *Mounter) GetFileType(pathname string) (FileType, error) {
-	var pathType FileType
-	info, err := os.Stat(pathname)
-	if os.IsNotExist(err) {
-		return pathType, fmt.Errorf("path %q does not exist", pathname)
-	}
-	// err in call to os.Stat
-	if err != nil {
-		return pathType, err
-	}
-
-	mode := info.Sys().(*syscall.Win32FileAttributeData).FileAttributes
-	switch mode & syscall.S_IFMT {
-	case syscall.S_IFSOCK:
-		return FileTypeSocket, nil
-	case syscall.S_IFBLK:
-		return FileTypeBlockDev, nil
-	case syscall.S_IFCHR:
-		return FileTypeCharDev, nil
-	case syscall.S_IFDIR:
-		return FileTypeDirectory, nil
-	case syscall.S_IFREG:
-		return FileTypeFile, nil
-	}
-
-	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
+	return getFileType(pathname)
 }
 
 // MakeFile creates a new directory
@@ -252,12 +238,125 @@ func (mounter *Mounter) MakeFile(pathname string) error {
 }
 
 // ExistsPath checks whether the path exists
-func (mounter *Mounter) ExistsPath(pathname string) bool {
-	_, err := os.Stat(pathname)
-	if err != nil {
-		return false
+func (mounter *Mounter) ExistsPath(pathname string) (bool, error) {
+	return utilfile.FileExists(pathname)
+}
+
+// check whether hostPath is within volume path
+// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
+func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(hostPath) == 0 {
+		return []uintptr{}, nil
 	}
-	return true
+
+	finalSubPath, err := filepath.EvalSymlinks(hostPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
+	}
+	finalVolumePath, err := filepath.EvalSymlinks(volumePath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
+	}
+
+	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
+}
+
+// lock all intermediate subPath directories and check they are all within volumePath
+// volumePath & subPath should not contain any symlink, otherwise it will return error
+func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(subPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	// get relative path to volumePath
+	relSubPath, err := filepath.Rel(volumePath, subPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
+	}
+	if startsWithBackstep(relSubPath) {
+		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
+	}
+
+	if relSubPath == "." {
+		// volumePath and subPath are equal
+		return []uintptr{}, nil
+	}
+
+	fileHandles := []uintptr{}
+	var errorResult error
+
+	currentFullPath := volumePath
+	dirs := strings.Split(relSubPath, string(os.PathSeparator))
+	for _, dir := range dirs {
+		// lock intermediate subPath directory first
+		currentFullPath = filepath.Join(currentFullPath, dir)
+		handle, err := lockPath(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
+			break
+		}
+		fileHandles = append(fileHandles, handle)
+
+		// make sure intermediate subPath directory does not contain symlink any more
+		stat, err := os.Lstat(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
+			break
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
+			break
+		}
+
+		if !pathWithinBase(currentFullPath, volumePath) {
+			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
+			break
+		}
+	}
+
+	return fileHandles, errorResult
+}
+
+// unlockPath unlock directories
+func unlockPath(fileHandles []uintptr) {
+	if fileHandles != nil {
+		for _, handle := range fileHandles {
+			syscall.CloseHandle(syscall.Handle(handle))
+		}
+	}
+}
+
+// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
+func lockPath(path string) (uintptr, error) {
+	if len(path) == 0 {
+		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return uintptr(syscall.InvalidHandle), err
+	}
+	access := uint32(syscall.GENERIC_READ)
+	sharemode := uint32(syscall.FILE_SHARE_READ)
+	createmode := uint32(syscall.OPEN_EXISTING)
+	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
+	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
+	return uintptr(fd), err
+}
+
+// Lock all directories in subPath and check they're not symlinks.
+func (mounter *Mounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
+
+	// Unlock the directories when the container starts
+	cleanupAction = func() {
+		unlockPath(handles)
+	}
+	return subPath.Path, cleanupAction, err
+}
+
+// No bind-mounts for subpaths are necessary on Windows
+func (mounter *Mounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
 }
 
 func (mounter *SafeFormatAndMount) formatAndMount(source string, target string, fstype string, options []string) error {
@@ -265,10 +364,23 @@ func (mounter *SafeFormatAndMount) formatAndMount(source string, target string,
 	glog.V(4).Infof("Attempting to formatAndMount disk: %s %s %s", fstype, source, target)
 
 	if err := ValidateDiskNumber(source); err != nil {
-		glog.Errorf("azureMount: formatAndMount failed, err: %v\n", err)
+		glog.Errorf("diskMount: formatAndMount failed, err: %v", err)
 		return err
 	}
 
+	if len(fstype) == 0 {
+		// Use 'NTFS' as the default
+		fstype = "NTFS"
+	}
+
+	// format disk if it is unformatted(raw)
+	cmd := fmt.Sprintf("Get-Disk -Number %s | Where partitionstyle -eq 'raw' | Initialize-Disk -PartitionStyle MBR -PassThru"+
+		" | New-Partition -AssignDriveLetter -UseMaximumSize | Format-Volume -FileSystem %s -Confirm:$false", source, fstype)
+	if output, err := mounter.Exec.Run("powershell", "/c", cmd); err != nil {
+		return fmt.Errorf("diskMount: format disk failed, error: %v, output: %q", err, string(output))
+	}
+	glog.V(4).Infof("diskMount: Disk successfully formatted, disk: %q, fstype: %q", source, fstype)
+
 	driveLetter, err := getDriveLetterByDiskNumber(source, mounter.Exec)
 	if err != nil {
 		return err
@@ -344,3 +456,153 @@ func getAllParentLinks(path string) ([]string, error) {
 
 	return links, nil
 }
+
+func (mounter *Mounter) GetMountRefs(pathname string) ([]string, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return nil, err
+	}
+	return getMountRefsByDev(mounter, realpath)
+}
+
+// Note that on windows, it always returns 0. We actually don't set FSGroup on
+// windows platform, see SetVolumeOwnership implementation.
+func (mounter *Mounter) GetFSGroup(pathname string) (int64, error) {
+	return 0, nil
+}
+
+func (mounter *Mounter) GetSELinuxSupport(pathname string) (bool, error) {
+	// Windows does not support SELinux.
+	return false, nil
+}
+
+func (mounter *Mounter) GetMode(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}
+
+// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
+func (mounter *Mounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	glog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !pathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			glog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	if len(toCreate) == 0 {
+		return nil
+	}
+
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
+	}
+	fullBasePath, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("cannot read link %s: %s", base, err)
+	}
+	if !pathWithinBase(fullExistingPath, fullBasePath) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
+	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
+	defer unlockPath(fileHandles)
+	if err != nil {
+		return err
+	}
+
+	glog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink by lock that directory immediately
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		glog.V(4).Infof("Creating %s", dir)
+		if err := os.Mkdir(currentPath, perm); err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		handle, err := lockPath(currentPath)
+		if err != nil {
+			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
+		}
+		defer syscall.CloseHandle(syscall.Handle(handle))
+		// make sure newly created directory does not contain symlink after lock
+		stat, err := os.Lstat(currentPath)
+		if err != nil {
+			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
+		}
+	}
+
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+
+	if startsWithBackstep(rel) {
+		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
+	}
+
+	if rel == "." {
+		// base and pathname are equal
+		return pathname, []string{}, nil
+	}
+
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	parent := base
+	currentPath := base
+	for i, dir := range dirs {
+		parent = currentPath
+		currentPath = filepath.Join(parent, dir)
+		if _, err := os.Lstat(currentPath); err != nil {
+			if os.IsNotExist(err) {
+				return parent, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+	}
+
+	return pathname, []string{}, nil
+}

vendor/k8s.io/kubernetes/pkg/util/mount/mount_windows_test.go

@@ -20,8 +20,14 @@ package mount
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestNormalizeWindowsPath(t *testing.T) {
@@ -132,3 +138,658 @@ func TestGetMountRefs(t *testing.T) {
 		}
 	}
 }
+
+func TestDoSafeMakeDir(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestDoSafeMakeDir")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+	os.MkdirAll(testingVolumePath, 0755)
+	defer os.RemoveAll(testingVolumePath)
+
+	tests := []struct {
+		volumePath    string
+		subPath       string
+		expectError   bool
+		symlinkTarget string
+	}{
+		{
+			volumePath:    testingVolumePath,
+			subPath:       ``,
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `x`),
+			expectError:   false,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectError:   false,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink`),
+			expectError:   false,
+			symlinkTarget: base,
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink\c\d`),
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `symlink\y926`),
+			expectError:   true,
+			symlinkTarget: "",
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\b\symlink`),
+			expectError:   false,
+			symlinkTarget: base,
+		},
+		{
+			volumePath:    testingVolumePath,
+			subPath:       filepath.Join(testingVolumePath, `a\x\symlink`),
+			expectError:   false,
+			symlinkTarget: filepath.Join(testingVolumePath, `a`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 && len(test.symlinkTarget) > 0 {
+			// make all parent sub directories
+			if parent := filepath.Dir(test.subPath); parent != "." {
+				os.MkdirAll(parent, 0755)
+			}
+
+			// make last element as symlink
+			linkPath := test.subPath
+			if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+				if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+					t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+				}
+			}
+		}
+
+		err := doSafeMakeDir(test.subPath, test.volumePath, os.FileMode(0755))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+		if _, err := os.Stat(test.subPath); os.IsNotExist(err) {
+			t.Errorf("subPath should exists after doSafeMakeDir(%s, %s)", test.subPath, test.volumePath)
+		}
+	}
+}
+
+func TestLockAndCheckSubPath(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestLockAndCheckSubPath")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		volumePath          string
+		subPath             string
+		expectedHandleCount int
+		expectError         bool
+		symlinkTarget       string
+	}{
+		{
+			volumePath:          `c:\`,
+			subPath:             ``,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          ``,
+			subPath:             `a`,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a`),
+			expectedHandleCount: 1,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectedHandleCount: 4,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `symlink`),
+			expectedHandleCount: 0,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
+			expectedHandleCount: 0,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
+			expectedHandleCount: 2,
+			expectError:         false,
+			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
+			os.MkdirAll(test.volumePath, 0755)
+			if len(test.symlinkTarget) == 0 {
+				// make all intermediate sub directories
+				os.MkdirAll(test.subPath, 0755)
+			} else {
+				// make all parent sub directories
+				if parent := filepath.Dir(test.subPath); parent != "." {
+					os.MkdirAll(parent, 0755)
+				}
+
+				// make last element as symlink
+				linkPath := test.subPath
+				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+					}
+				}
+			}
+		}
+
+		fileHandles, err := lockAndCheckSubPath(test.volumePath, test.subPath)
+		unlockPath(fileHandles)
+		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+	}
+
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}
+
+func TestLockAndCheckSubPathWithoutSymlink(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestLockAndCheckSubPathWithoutSymlink")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		volumePath          string
+		subPath             string
+		expectedHandleCount int
+		expectError         bool
+		symlinkTarget       string
+	}{
+		{
+			volumePath:          `c:\`,
+			subPath:             ``,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          ``,
+			subPath:             `a`,
+			expectedHandleCount: 0,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a`),
+			expectedHandleCount: 1,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectedHandleCount: 4,
+			expectError:         false,
+			symlinkTarget:       "",
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `symlink`),
+			expectedHandleCount: 1,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\symlink`),
+			expectedHandleCount: 4,
+			expectError:         true,
+			symlinkTarget:       base,
+		},
+		{
+			volumePath:          testingVolumePath,
+			subPath:             filepath.Join(testingVolumePath, `a\b\c\d\symlink`),
+			expectedHandleCount: 5,
+			expectError:         true,
+			symlinkTarget:       filepath.Join(testingVolumePath, `a\b`),
+		},
+	}
+
+	for _, test := range tests {
+		if len(test.volumePath) > 0 && len(test.subPath) > 0 {
+			os.MkdirAll(test.volumePath, 0755)
+			if len(test.symlinkTarget) == 0 {
+				// make all intermediate sub directories
+				os.MkdirAll(test.subPath, 0755)
+			} else {
+				// make all parent sub directories
+				if parent := filepath.Dir(test.subPath); parent != "." {
+					os.MkdirAll(parent, 0755)
+				}
+
+				// make last element as symlink
+				linkPath := test.subPath
+				if _, err := os.Stat(linkPath); err != nil && os.IsNotExist(err) {
+					if err := makeLink(linkPath, test.symlinkTarget); err != nil {
+						t.Fatalf("unexpected error: %v", fmt.Errorf("mklink link(%q) target(%q) error: %q", linkPath, test.symlinkTarget, err))
+					}
+				}
+			}
+		}
+
+		fileHandles, err := lockAndCheckSubPathWithoutSymlink(test.volumePath, test.subPath)
+		unlockPath(fileHandles)
+		assert.Equal(t, test.expectedHandleCount, len(fileHandles))
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during LockAndCheckSubPath(%s, %s)", test.volumePath, test.subPath)
+	}
+
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}
+
+func TestFindExistingPrefix(t *testing.T) {
+	base, err := ioutil.TempDir("", "TestFindExistingPrefix")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+
+	defer os.RemoveAll(base)
+
+	testingVolumePath := filepath.Join(base, "testingVolumePath")
+
+	tests := []struct {
+		base                    string
+		pathname                string
+		expectError             bool
+		expectedExistingPath    string
+		expectedToCreateDirs    []string
+		createSubPathBeforeTest bool
+	}{
+		{
+			base:                    `c:\tmp\a`,
+			pathname:                `c:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    ``,
+			pathname:                `c:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    `c:\tmp\a`,
+			pathname:                `d:\tmp\b`,
+			expectError:             true,
+			expectedExistingPath:    "",
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                testingVolumePath,
+			expectError:             false,
+			expectedExistingPath:    testingVolumePath,
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{},
+			createSubPathBeforeTest: true,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b\c\`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{`c`},
+			createSubPathBeforeTest: false,
+		},
+		{
+			base:                    testingVolumePath,
+			pathname:                filepath.Join(testingVolumePath, `a\b\c\d`),
+			expectError:             false,
+			expectedExistingPath:    filepath.Join(testingVolumePath, `a\b`),
+			expectedToCreateDirs:    []string{`c`, `d`},
+			createSubPathBeforeTest: false,
+		},
+	}
+
+	for _, test := range tests {
+		if test.createSubPathBeforeTest {
+			os.MkdirAll(test.pathname, 0755)
+		}
+
+		existingPath, toCreate, err := findExistingPrefix(test.base, test.pathname)
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during findExistingPrefix(%s, %s)", test.base, test.pathname)
+			continue
+		}
+		assert.Nil(t, err, "Expect no error during findExistingPrefix(%s, %s)", test.base, test.pathname)
+
+		assert.Equal(t, test.expectedExistingPath, existingPath, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
+			test.base, test.pathname, existingPath, test.expectedExistingPath)
+
+		assert.Equal(t, test.expectedToCreateDirs, toCreate, "Expect result not equal with findExistingPrefix(%s, %s) return: %q, expected: %q",
+			test.base, test.pathname, toCreate, test.expectedToCreateDirs)
+
+	}
+	// remove dir will happen after closing all file handles
+	assert.Nil(t, os.RemoveAll(testingVolumePath), "Expect no error during remove dir %s", testingVolumePath)
+}
+
+func TestPathWithinBase(t *testing.T) {
+	tests := []struct {
+		fullPath       string
+		basePath       string
+		expectedResult bool
+	}{
+		{
+			fullPath:       `c:\tmp\a\b\c`,
+			basePath:       `c:\tmp`,
+			expectedResult: true,
+		},
+		{
+			fullPath:       `c:\tmp1`,
+			basePath:       `c:\tmp2`,
+			expectedResult: false,
+		},
+		{
+			fullPath:       `c:\tmp`,
+			basePath:       `c:\tmp`,
+			expectedResult: true,
+		},
+		{
+			fullPath:       `c:\tmp`,
+			basePath:       `c:\tmp\a\b\c`,
+			expectedResult: false,
+		},
+		{
+			fullPath:       `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config\..timestamp\file.txt`,
+			basePath:       `c:\kubelet\pods\uuid\volumes\kubernetes.io~configmap\config`,
+			expectedResult: true,
+		},
+	}
+
+	for _, test := range tests {
+		result := pathWithinBase(test.fullPath, test.basePath)
+		assert.Equal(t, result, test.expectedResult, "Expect result not equal with pathWithinBase(%s, %s) return: %q, expected: %q",
+			test.fullPath, test.basePath, result, test.expectedResult)
+	}
+}
+
+func TestGetFileType(t *testing.T) {
+	mounter := New("fake/path")
+
+	testCase := []struct {
+		name         string
+		expectedType FileType
+		setUp        func() (string, string, error)
+	}{
+		{
+			"Directory Test",
+			FileTypeDirectory,
+			func() (string, string, error) {
+				tempDir, err := ioutil.TempDir("", "test-get-filetype-")
+				return tempDir, tempDir, err
+			},
+		},
+		{
+			"File Test",
+			FileTypeFile,
+			func() (string, string, error) {
+				tempFile, err := ioutil.TempFile("", "test-get-filetype")
+				if err != nil {
+					return "", "", err
+				}
+				tempFile.Close()
+				return tempFile.Name(), tempFile.Name(), nil
+			},
+		},
+	}
+
+	for idx, tc := range testCase {
+		path, cleanUpPath, err := tc.setUp()
+		if err != nil {
+			t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err)
+		}
+		if len(cleanUpPath) > 0 {
+			defer os.RemoveAll(cleanUpPath)
+		}
+
+		fileType, err := mounter.GetFileType(path)
+		if err != nil {
+			t.Fatalf("[%d-%s] unexpected error : %v", idx, tc.name, err)
+		}
+		if fileType != tc.expectedType {
+			t.Fatalf("[%d-%s] expected %s, but got %s", idx, tc.name, tc.expectedType, fileType)
+		}
+	}
+}
+
+func TestIsLikelyNotMountPoint(t *testing.T) {
+	mounter := Mounter{"fake/path"}
+
+	tests := []struct {
+		fileName       string
+		targetLinkName string
+		setUp          func(base, fileName, targetLinkName string) error
+		expectedResult bool
+		expectError    bool
+	}{
+		{
+			"Dir",
+			"",
+			func(base, fileName, targetLinkName string) error {
+				return os.Mkdir(filepath.Join(base, fileName), 0750)
+			},
+			true,
+			false,
+		},
+		{
+			"InvalidDir",
+			"",
+			func(base, fileName, targetLinkName string) error {
+				return nil
+			},
+			true,
+			true,
+		},
+		{
+			"ValidSymLink",
+			"targetSymLink",
+			func(base, fileName, targetLinkName string) error {
+				targeLinkPath := filepath.Join(base, targetLinkName)
+				if err := os.Mkdir(targeLinkPath, 0750); err != nil {
+					return err
+				}
+
+				filePath := filepath.Join(base, fileName)
+				if err := makeLink(filePath, targeLinkPath); err != nil {
+					return err
+				}
+				return nil
+			},
+			false,
+			false,
+		},
+		{
+			"InvalidSymLink",
+			"targetSymLink2",
+			func(base, fileName, targetLinkName string) error {
+				targeLinkPath := filepath.Join(base, targetLinkName)
+				if err := os.Mkdir(targeLinkPath, 0750); err != nil {
+					return err
+				}
+
+				filePath := filepath.Join(base, fileName)
+				if err := makeLink(filePath, targeLinkPath); err != nil {
+					return err
+				}
+				return removeLink(targeLinkPath)
+			},
+			true,
+			false,
+		},
+	}
+
+	for _, test := range tests {
+		base, err := ioutil.TempDir("", test.fileName)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+
+		defer os.RemoveAll(base)
+
+		if err := test.setUp(base, test.fileName, test.targetLinkName); err != nil {
+			t.Fatalf("unexpected error in setUp(%s, %s): %v", test.fileName, test.targetLinkName, err)
+		}
+
+		filePath := filepath.Join(base, test.fileName)
+		result, err := mounter.IsLikelyNotMountPoint(filePath)
+		assert.Equal(t, result, test.expectedResult, "Expect result not equal with IsLikelyNotMountPoint(%s) return: %q, expected: %q",
+			filePath, result, test.expectedResult)
+
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during IsLikelyNotMountPoint(%s)", filePath)
+		} else {
+			assert.Nil(t, err, "Expect error is nil during IsLikelyNotMountPoint(%s)", filePath)
+		}
+	}
+}
+
+func TestFormatAndMount(t *testing.T) {
+	fakeMounter := ErrorMounter{&FakeMounter{}, 0, nil}
+	execCallback := func(cmd string, args ...string) ([]byte, error) {
+		for j := range args {
+			if strings.Contains(args[j], "Get-Disk -Number") {
+				return []byte("0"), nil
+			}
+
+			if strings.Contains(args[j], "Get-Partition -DiskNumber") {
+				return []byte("0"), nil
+			}
+
+			if strings.Contains(args[j], "mklink") {
+				return nil, nil
+			}
+		}
+		return nil, fmt.Errorf("Unexpected cmd %s, args %v", cmd, args)
+	}
+	fakeExec := NewFakeExec(execCallback)
+
+	mounter := SafeFormatAndMount{
+		Interface: &fakeMounter,
+		Exec:      fakeExec,
+	}
+
+	tests := []struct {
+		device       string
+		target       string
+		fstype       string
+		mountOptions []string
+		expectError  bool
+	}{
+		{
+			"0",
+			"disk",
+			"NTFS",
+			[]string{},
+			false,
+		},
+		{
+			"0",
+			"disk",
+			"",
+			[]string{},
+			false,
+		},
+		{
+			"invalidDevice",
+			"disk",
+			"NTFS",
+			[]string{},
+			true,
+		},
+	}
+
+	for _, test := range tests {
+		base, err := ioutil.TempDir("", test.device)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		defer os.RemoveAll(base)
+
+		target := filepath.Join(base, test.target)
+		err = mounter.FormatAndMount(test.device, target, test.fstype, test.mountOptions)
+		if test.expectError {
+			assert.NotNil(t, err, "Expect error during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions)
+		} else {
+			assert.Nil(t, err, "Expect error is nil during FormatAndMount(%s, %s, %s, %v)", test.device, test.target, test.fstype, test.mountOptions)
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount.go

@@ -23,8 +23,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 
 	"github.com/golang/glog"
+	"golang.org/x/sys/unix"
+	utilfile "k8s.io/kubernetes/pkg/util/file"
 	"k8s.io/kubernetes/pkg/util/nsenter"
 )
 
@@ -40,10 +43,16 @@ const (
 // the host's mount namespace.
 type NsenterMounter struct {
 	ne *nsenter.Nsenter
+	// rootDir is location of /var/lib/kubelet directory.
+	rootDir string
 }
 
-func NewNsenterMounter() *NsenterMounter {
-	return &NsenterMounter{ne: nsenter.NewNsenter()}
+// NewNsenterMounter creates a new mounter for kubelet that runs as a container.
+func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
+	return &NsenterMounter{
+		rootDir: rootDir,
+		ne:      ne,
+	}
 }
 
 // NsenterMounter implements mount.Interface
@@ -225,8 +234,13 @@ func (n *NsenterMounter) MakeRShared(path string) error {
 
 func (mounter *NsenterMounter) GetFileType(pathname string) (FileType, error) {
 	var pathType FileType
-	outputBytes, err := mounter.ne.Exec("stat", []string{"-L", `--printf "%F"`, pathname}).CombinedOutput()
+	outputBytes, err := mounter.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput()
 	if err != nil {
+		if strings.Contains(string(outputBytes), "No such file") {
+			err = fmt.Errorf("%s does not exist", pathname)
+		} else {
+			err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes))
+		}
 		return pathType, err
 	}
 
@@ -262,11 +276,177 @@ func (mounter *NsenterMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *NsenterMounter) ExistsPath(pathname string) bool {
-	args := []string{pathname}
-	_, err := mounter.ne.Exec("ls", args).CombinedOutput()
-	if err == nil {
-		return true
+func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
+	// Resolve the symlinks but allow the target not to exist. EvalSymlinks
+	// would return an generic error when the target does not exist.
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */)
+	if err != nil {
+		return false, err
 	}
-	return false
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
+	return utilfile.FileExists(kubeletpath)
+}
+
+func (mounter *NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(mounter, podDir, volumeName)
+}
+
+func (mounter *NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	// Bind-mount the subpath to avoid using symlinks in subpaths.
+	newHostPath, err = doNsEnterBindSubPath(mounter, subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+func (mounter *NsenterMounter) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	fullSubdirPath := filepath.Join(base, subdir)
+	evaluatedSubdirPath, err := mounter.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
+	}
+	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
+
+	evaluatedBase, err := mounter.ne.EvalSymlinks(base, true /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+	evaluatedBase = filepath.Clean(evaluatedBase)
+
+	rootDir := filepath.Clean(mounter.rootDir)
+	if pathWithinBase(evaluatedBase, rootDir) {
+		// Base is in /var/lib/kubelet. This directory is shared between the
+		// container with kubelet and the host. We don't need to add '/rootfs'.
+		// This is useful when /rootfs is mounted as read-only - we can still
+		// create subpaths for paths in /var/lib/kubelet.
+		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
+	}
+
+	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
+	// the directory there.
+	// This requires /rootfs to be writable.
+	kubeletSubdirPath := mounter.ne.KubeletPath(evaluatedSubdirPath)
+	kubeletBase := mounter.ne.KubeletPath(evaluatedBase)
+	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
+}
+
+func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
+	hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
+	if err != nil {
+		return nil, err
+	}
+	return searchMountPoints(hostpath, hostProcMountinfoPath)
+}
+
+func doNsEnterBindSubPath(mounter *NsenterMounter, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs in a container:
+	// - safely open the subpath
+	// - bind-mount the subpath to target (this can be unsafe)
+	// - check that we mounted the right thing by comparing device ID and inode
+	//   of the subpath (via safely opened fd) and the target (that's under our
+	//   control)
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	evaluatedHostVolumePath, err := mounter.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	evaluatedHostSubpath, err := mounter.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	glog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
+	subpath.VolumePath = mounter.ne.KubeletPath(evaluatedHostVolumePath)
+	subpath.Path = mounter.ne.KubeletPath(evaluatedHostSubpath)
+
+	// Check the subpath is correct and open it
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			glog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				glog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	// Leap of faith: optimistically expect that nobody has modified previously
+	// expanded evalSubPath with evil symlinks and bind-mount it.
+	// Mount is done on the host! don't use kubelet path!
+	glog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
+	if err = mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
+	}
+
+	// Check that the bind-mount target is the same inode and device as the
+	// source that we keept open, i.e. we mounted the right thing.
+	err = checkDeviceInode(fd, bindPathTarget)
+	if err != nil {
+		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
+	}
+
+	success = true
+	glog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// checkDeviceInode checks that opened file and path represent the same file.
+func checkDeviceInode(fd int, path string) error {
+	var srcStat, dstStat unix.Stat_t
+	err := unix.Fstat(fd, &srcStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on subpath FD: %v", err)
+	}
+
+	err = unix.Stat(path, &dstStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on %s: %v", path, err)
+	}
+
+	if srcStat.Dev != dstStat.Dev {
+		return fmt.Errorf("different device number")
+	}
+	if srcStat.Ino != dstStat.Ino {
+		return fmt.Errorf("different inode")
+	}
+	return nil
+}
+
+func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
+	if err != nil {
+		return -1, err
+	}
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
+	return getFSGroup(kubeletpath)
+}
+
+func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return getSELinuxSupport(pathname, hostProcMountsPath)
+}
+
+func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
+	hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
+	if err != nil {
+		return 0, err
+	}
+	kubeletpath := mounter.ne.KubeletPath(hostPath)
+	return getMode(kubeletpath)
 }

vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_test.go

@@ -18,7 +18,16 @@ limitations under the License.
 
 package mount
 
-import "testing"
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/kubernetes/pkg/util/nsenter"
+)
 
 func TestParseFindMnt(t *testing.T) {
 	tests := []struct {
@@ -65,3 +74,636 @@ func TestParseFindMnt(t *testing.T) {
 		}
 	}
 }
+
+func TestCheckDeviceInode(t *testing.T) {
+	testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
+	if err != nil {
+		t.Fatalf("Cannot create temporary directory: %s", err)
+	}
+	defer os.RemoveAll(testDir)
+
+	tests := []struct {
+		name        string
+		srcPath     string
+		dstPath     string
+		expectError string
+	}{
+		{
+			name:        "the same file",
+			srcPath:     filepath.Join(testDir, "1"),
+			dstPath:     filepath.Join(testDir, "1"),
+			expectError: "",
+		},
+		{
+			name:        "different file on the same FS",
+			srcPath:     filepath.Join(testDir, "2.1"),
+			dstPath:     filepath.Join(testDir, "2.2"),
+			expectError: "different inode",
+		},
+		{
+			name:    "different file on different device",
+			srcPath: filepath.Join(testDir, "3"),
+			// /proc is always on a different "device" than /tmp (or $TEMP)
+			dstPath:     "/proc/self/status",
+			expectError: "different device",
+		},
+	}
+
+	for _, test := range tests {
+		if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
+			t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
+			continue
+		}
+
+		// Don't create dst if it exists
+		if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
+			if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
+				t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
+				continue
+			}
+		} else if err != nil {
+			t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
+			continue
+		}
+
+		fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
+		if err != nil {
+			t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
+			continue
+		}
+
+		err = checkDeviceInode(fd, test.dstPath)
+
+		if test.expectError == "" && err != nil {
+			t.Errorf("Test %q: expected no error, got %s", test.name, err)
+		}
+		if test.expectError != "" {
+			if err == nil {
+				t.Errorf("Test %q: expected error, got none", test.name)
+			} else {
+				if !strings.Contains(err.Error(), test.expectError) {
+					t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
+				}
+			}
+		}
+	}
+}
+
+func newFakeNsenterMounter(tmpdir string, t *testing.T) (mounter *NsenterMounter, rootfsPath string, varlibPath string, err error) {
+	rootfsPath = filepath.Join(tmpdir, "rootfs")
+	if err := os.Mkdir(rootfsPath, 0755); err != nil {
+		return nil, "", "", err
+	}
+	ne, err := nsenter.NewFakeNsenter(rootfsPath)
+	if err != nil {
+		return nil, "", "", err
+	}
+
+	varlibPath = filepath.Join(tmpdir, "/var/lib/kubelet")
+	if err := os.MkdirAll(varlibPath, 0755); err != nil {
+		return nil, "", "", err
+	}
+
+	return NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, nil
+}
+
+func TestNsenterExistsFile(t *testing.T) {
+	tests := []struct {
+		name           string
+		prepare        func(base, rootfs string) (string, error)
+		expectedOutput bool
+		expectError    bool
+	}{
+		{
+			name: "simple existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/file
+				path := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
+					return "", err
+				}
+				// In kubelet: /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, path, 0644); err != nil {
+					return "", err
+				}
+				return path, nil
+			},
+			expectedOutput: true,
+		},
+		{
+			name: "simple non-existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				path := filepath.Join(base, "file")
+				return path, nil
+			},
+			expectedOutput: false,
+		},
+		{
+			name: "simple non-accessible file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host:
+				// create /base/dir/file, then make the dir inaccessible
+				dir := filepath.Join(base, "dir")
+				if err := os.MkdirAll(dir, 0755); err != nil {
+					return "", err
+				}
+				path := filepath.Join(dir, "file")
+				if err := ioutil.WriteFile(path, []byte{}, 0); err != nil {
+					return "", err
+				}
+				if err := os.Chmod(dir, 0644); err != nil {
+					return "", err
+				}
+
+				// In kubelet: do the same with /rootfs/base/dir/file
+				rootfsPath, err := writeRootfsFile(rootfs, path, 0777)
+				if err != nil {
+					return "", err
+				}
+				rootfsDir := filepath.Dir(rootfsPath)
+				if err := os.Chmod(rootfsDir, 0644); err != nil {
+					return "", err
+				}
+
+				return path, nil
+			},
+			expectedOutput: false,
+			expectError:    true,
+		},
+		{
+			name: "relative symlink to existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/link -> file
+				file := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
+					return "", err
+				}
+				path := filepath.Join(base, "link")
+				if err := os.Symlink("file", path); err != nil {
+					return "", err
+				}
+				// In kubelet: /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
+					return "", err
+				}
+				return path, nil
+			},
+			expectedOutput: true,
+		},
+		{
+			name: "absolute symlink to existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/link -> /base/file
+				file := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(file, []byte{}, 0); err != nil {
+					return "", err
+				}
+				path := filepath.Join(base, "link")
+				if err := os.Symlink(file, path); err != nil {
+					return "", err
+				}
+				// In kubelet: /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, file, 0644); err != nil {
+					return "", err
+				}
+
+				return path, nil
+			},
+			expectedOutput: true,
+		},
+		{
+			name: "relative symlink to non-existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				path := filepath.Join(base, "link")
+				if err := os.Symlink("file", path); err != nil {
+					return "", err
+				}
+				return path, nil
+			},
+			expectedOutput: false,
+		},
+		{
+			name: "absolute symlink to non-existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				file := filepath.Join(base, "file")
+				path := filepath.Join(base, "link")
+				if err := os.Symlink(file, path); err != nil {
+					return "", err
+				}
+				return path, nil
+			},
+			expectedOutput: false,
+		},
+		{
+			name: "symlink loop",
+			prepare: func(base, rootfs string) (string, error) {
+				path := filepath.Join(base, "link")
+				if err := os.Symlink(path, path); err != nil {
+					return "", err
+				}
+				return path, nil
+			},
+			expectedOutput: false,
+			// TODO: realpath -m is not able to detect symlink loop. Should we care?
+			expectError: false,
+		},
+	}
+
+	for _, test := range tests {
+		tmpdir, err := ioutil.TempDir("", "nsenter-exists-file")
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		defer os.RemoveAll(tmpdir)
+
+		testBase := filepath.Join(tmpdir, "base")
+		if err := os.Mkdir(testBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+
+		mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		path, err := test.prepare(testBase, rootfs)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		out, err := mounter.ExistsPath(path)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+
+		if out != test.expectedOutput {
+			t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedOutput, out)
+		}
+	}
+}
+
+func TestNsenterGetMode(t *testing.T) {
+	tests := []struct {
+		name         string
+		prepare      func(base, rootfs string) (string, error)
+		expectedMode os.FileMode
+		expectError  bool
+	}{
+		{
+			name: "simple file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/file
+				path := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(path, []byte{}, 0644); err != nil {
+					return "", err
+				}
+
+				// Prepare a different file as /rootfs/base/file (="the host
+				// visible from container") to check that NsEnterMounter calls
+				// stat on this file and not on /base/file.
+				// Visible from kubelet: /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, path, 0777); err != nil {
+					return "", err
+				}
+
+				return path, nil
+			},
+			expectedMode: 0777,
+		},
+		{
+			name: "non-existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				path := filepath.Join(base, "file")
+				return path, nil
+			},
+			expectedMode: 0,
+			expectError:  true,
+		},
+		{
+			name: "absolute symlink to existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/link -> /base/file
+				file := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(file, []byte{}, 0644); err != nil {
+					return "", err
+				}
+				path := filepath.Join(base, "link")
+				if err := os.Symlink(file, path); err != nil {
+					return "", err
+				}
+
+				// Visible from kubelet:
+				// /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, file, 0747); err != nil {
+					return "", err
+				}
+
+				return path, nil
+			},
+			expectedMode: 0747,
+		},
+		{
+			name: "relative symlink to existing file",
+			prepare: func(base, rootfs string) (string, error) {
+				// On the host: /base/link -> file
+				file := filepath.Join(base, "file")
+				if err := ioutil.WriteFile(file, []byte{}, 0741); err != nil {
+					return "", err
+				}
+				path := filepath.Join(base, "link")
+				if err := os.Symlink("file", path); err != nil {
+					return "", err
+				}
+
+				// Visible from kubelet:
+				// /rootfs/base/file
+				if _, err := writeRootfsFile(rootfs, file, 0647); err != nil {
+					return "", err
+				}
+
+				return path, nil
+			},
+			expectedMode: 0647,
+		},
+	}
+
+	for _, test := range tests {
+		tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		defer os.RemoveAll(tmpdir)
+
+		testBase := filepath.Join(tmpdir, "base")
+		if err := os.Mkdir(testBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+
+		mounter, rootfs, _, err := newFakeNsenterMounter(tmpdir, t)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		path, err := test.prepare(testBase, rootfs)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		mode, err := mounter.GetMode(path)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+
+		if mode != test.expectedMode {
+			t.Errorf("Test %q: expected return value %v, got %v", test.name, test.expectedMode, mode)
+		}
+	}
+}
+
+func writeRootfsFile(rootfs, path string, mode os.FileMode) (string, error) {
+	fullPath := filepath.Join(rootfs, path)
+	dir := filepath.Dir(fullPath)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return "", err
+	}
+	if err := ioutil.WriteFile(fullPath, []byte{}, mode); err != nil {
+		return "", err
+	}
+	// Use chmod, io.WriteFile is affected by umask
+	if err := os.Chmod(fullPath, mode); err != nil {
+		return "", err
+	}
+	return fullPath, nil
+}
+
+func TestNsenterSafeMakeDir(t *testing.T) {
+	tests := []struct {
+		name        string
+		prepare     func(base, rootfs, varlib string) (expectedDir string, err error)
+		subdir      string
+		expectError bool
+		// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
+		baseIsVarLib bool
+	}{
+		{
+			name: "simple directory",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// expected to be created in /roots/
+				expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "simple existing directory",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: directory exists
+				hostPath := filepath.Join(base, "some/subdirectory/structure")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				// In rootfs: directory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected to be created in /roots/
+				expectedDir = kubeletPath
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "absolute symlink into safe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
+				hostPath := filepath.Join(base, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(base, "some")
+				otherPath := filepath.Join(base, "other")
+				if err := os.Symlink(otherPath, somePath); err != nil {
+					return "", err
+				}
+
+				// In rootfs: /base/other/subdirectory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected 'structure' to be created
+				expectedDir = filepath.Join(rootfs, hostPath, "structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "relative symlink into safe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/other/subdirectory exists, /base/some is link to other
+				hostPath := filepath.Join(base, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(base, "some")
+				if err := os.Symlink("other", somePath); err != nil {
+					return "", err
+				}
+
+				// In rootfs: /base/other/subdirectory exists
+				kubeletPath := filepath.Join(rootfs, hostPath)
+				if err := os.MkdirAll(kubeletPath, 0755); err != nil {
+					return "", err
+				}
+				// expected 'structure' to be created
+				expectedDir = filepath.Join(rootfs, hostPath, "structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "symlink into unsafe place",
+			// evaluated in base
+			subdir: "some/subdirectory/structure",
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /base/some is link to /bin/other
+				somePath := filepath.Join(base, "some")
+				if err := os.Symlink("/bin", somePath); err != nil {
+					return "", err
+				}
+				return "", nil
+			},
+			expectError: true,
+		},
+		{
+			name: "simple directory in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
+				expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "safe symlink in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
+				hostPath := filepath.Join(varlib, "other/subdirectory")
+				if err := os.MkdirAll(hostPath, 0755); err != nil {
+					return "", err
+				}
+				somePath := filepath.Join(varlib, "some")
+				if err := os.Symlink("other", somePath); err != nil {
+					return "", err
+				}
+
+				// expected to be created in /base/var/lib/kubelet, not in /rootfs!
+				expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
+				return expectedDir, nil
+			},
+		},
+		{
+			name: "unsafe symlink in /var/lib/kubelet",
+			// evaluated in varlib
+			subdir:       "some/subdirectory/structure",
+			baseIsVarLib: true,
+			prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
+				// On the host: /varlib/some is link to /bin
+				somePath := filepath.Join(varlib, "some")
+				if err := os.Symlink("/bin", somePath); err != nil {
+					return "", err
+				}
+
+				return "", nil
+			},
+			expectError: true,
+		},
+	}
+	for _, test := range tests {
+		tmpdir, err := ioutil.TempDir("", "nsenter-get-mode-")
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		defer os.RemoveAll(tmpdir)
+
+		mounter, rootfs, varlib, err := newFakeNsenterMounter(tmpdir, t)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		// Prepare base directory for the test
+		testBase := filepath.Join(tmpdir, "base")
+		if err := os.Mkdir(testBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+		// Prepare base directory also in /rootfs
+		rootfsBase := filepath.Join(rootfs, testBase)
+		if err := os.MkdirAll(rootfsBase, 0755); err != nil {
+			t.Error(err)
+			continue
+		}
+
+		expectedDir := ""
+		if test.prepare != nil {
+			expectedDir, err = test.prepare(testBase, rootfs, varlib)
+			if err != nil {
+				t.Error(err)
+				continue
+			}
+		}
+
+		if test.baseIsVarLib {
+			// use /var/lib/kubelet as the test base so we can test creating
+			// subdirs there directly in /var/lib/kubenet and not in
+			// /rootfs/var/lib/kubelet
+			testBase = varlib
+		}
+
+		err = mounter.SafeMakeDir(test.subdir, testBase, 0755)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if test.expectError {
+			if err == nil {
+				t.Errorf("Test %q: expected error, got none", test.name)
+			} else {
+				if !strings.Contains(err.Error(), "is outside of allowed base") {
+					t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
+				}
+			}
+		}
+
+		if expectedDir != "" {
+			_, err := os.Stat(expectedDir)
+			if err != nil {
+				t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
+			}
+		}
+	}
+}

vendor/k8s.io/kubernetes/pkg/util/mount/nsenter_mount_unsupported.go

@@ -20,11 +20,14 @@ package mount
 
 import (
 	"errors"
+	"os"
+
+	"k8s.io/kubernetes/pkg/util/nsenter"
 )
 
 type NsenterMounter struct{}
 
-func NewNsenterMounter() *NsenterMounter {
+func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
 	return &NsenterMounter{}
 }
 
@@ -82,6 +85,34 @@ func (*NsenterMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (*NsenterMounter) ExistsPath(pathname string) bool {
-	return true
+func (*NsenterMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
+}
+
+func (*NsenterMounter) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}
+
+func (*NsenterMounter) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+func (*NsenterMounter) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
+func (*NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (*NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
 }

vendor/k8s.io/kubernetes/pkg/util/mount/safe_format_and_mount_test.go

@@ -116,7 +116,7 @@ func TestSafeFormatAndMount(t *testing.T) {
 			execScripts: []ExecArgs{
 				{"fsck", []string{"-a", "/dev/foo"}, "", nil},
 				{"blkid", []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", "/dev/foo"}, "", &fakeexec.FakeExitError{Status: 2}},
-				{"mkfs.ext4", []string{"-F", "/dev/foo"}, "", fmt.Errorf("formatting failed")},
+				{"mkfs.ext4", []string{"-F", "-m0", "/dev/foo"}, "", fmt.Errorf("formatting failed")},
 			},
 			expectedError: fmt.Errorf("formatting failed"),
 		},
@@ -127,7 +127,7 @@ func TestSafeFormatAndMount(t *testing.T) {
 			execScripts: []ExecArgs{
 				{"fsck", []string{"-a", "/dev/foo"}, "", nil},
 				{"blkid", []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", "/dev/foo"}, "", &fakeexec.FakeExitError{Status: 2}},
-				{"mkfs.ext4", []string{"-F", "/dev/foo"}, "", nil},
+				{"mkfs.ext4", []string{"-F", "-m0", "/dev/foo"}, "", nil},
 			},
 			expectedError: fmt.Errorf("Still cannot mount"),
 		},
@@ -138,7 +138,7 @@ func TestSafeFormatAndMount(t *testing.T) {
 			execScripts: []ExecArgs{
 				{"fsck", []string{"-a", "/dev/foo"}, "", nil},
 				{"blkid", []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", "/dev/foo"}, "", &fakeexec.FakeExitError{Status: 2}},
-				{"mkfs.ext4", []string{"-F", "/dev/foo"}, "", nil},
+				{"mkfs.ext4", []string{"-F", "-m0", "/dev/foo"}, "", nil},
 			},
 			expectedError: nil,
 		},
@@ -149,7 +149,7 @@ func TestSafeFormatAndMount(t *testing.T) {
 			execScripts: []ExecArgs{
 				{"fsck", []string{"-a", "/dev/foo"}, "", nil},
 				{"blkid", []string{"-p", "-s", "TYPE", "-s", "PTTYPE", "-o", "export", "/dev/foo"}, "", &fakeexec.FakeExitError{Status: 2}},
-				{"mkfs.ext3", []string{"-F", "/dev/foo"}, "", nil},
+				{"mkfs.ext3", []string{"-F", "-m0", "/dev/foo"}, "", nil},
 			},
 			expectedError: nil,
 		},

vendor/k8s.io/kubernetes/pkg/util/nsenter/BUILD

@@ -1,4 +1,4 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -92,3 +92,20 @@ filegroup(
     tags = ["automanaged"],
     visibility = ["//visibility:public"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "nsenter_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    embed = [":go_default_library"],
+    deps = select({
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//vendor/k8s.io/utils/exec:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
+)

vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter.go

@@ -19,9 +19,12 @@ limitations under the License.
 package nsenter
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"k8s.io/utils/exec"
 
@@ -29,9 +32,11 @@ import (
 )
 
 const (
-	hostRootFsPath = "/rootfs"
-	// hostProcMountNsPath is the default mount namespace for rootfs
-	hostProcMountNsPath = "/rootfs/proc/1/ns/mnt"
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+	// mountNsPath is the default mount namespace of the host
+	mountNsPath = "/proc/1/ns/mnt"
 	// nsenterPath is the default nsenter command
 	nsenterPath = "nsenter"
 )
@@ -57,55 +62,73 @@ const (
 //     contents. TODO: remove this requirement.
 // 6.  The host image must have "mount", "findmnt", "umount", "stat", "touch",
 //     "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
-// 7.  The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin
+// 7.  The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
+//     systemd is installed/enabled in the operating system.
 // For more information about mount propagation modes, see:
 //   https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
 type Nsenter struct {
 	// a map of commands to their paths on the host filesystem
 	paths map[string]string
+
+	// Path to the host filesystem, typically "/rootfs". Used only for testing.
+	hostRootFsPath string
+
+	// Exec implementation, used only for testing
+	executor exec.Interface
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() *Nsenter {
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
 	ne := &Nsenter{
-		paths: map[string]string{
-			"mount":       "",
-			"findmnt":     "",
-			"umount":      "",
-			"systemd-run": "",
-			"stat":        "",
-			"touch":       "",
-			"mkdir":       "",
-			"ls":          "",
-			"sh":          "",
-			"chmod":       "",
-		},
+		hostRootFsPath: hostRootFsPath,
+		executor:       executor,
+	}
+	if err := ne.initPaths(); err != nil {
+		return nil, err
+	}
+	return ne, nil
+}
+
+func (ne *Nsenter) initPaths() error {
+	ne.paths = map[string]string{}
+	binaries := []string{
+		"mount",
+		"findmnt",
+		"umount",
+		"systemd-run",
+		"stat",
+		"touch",
+		"mkdir",
+		"sh",
+		"chmod",
+		"realpath",
 	}
 	// search for the required commands in other locations besides /usr/bin
-	for binary := range ne.paths {
-		// default to root
-		ne.paths[binary] = filepath.Join("/", binary)
-		for _, path := range []string{"/bin", "/usr/sbin", "/usr/bin"} {
+	for _, binary := range binaries {
+		// check for binary under the following directories
+		for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
 			binPath := filepath.Join(path, binary)
-			if _, err := os.Stat(filepath.Join(hostRootFsPath, binPath)); err != nil {
+			if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
 				continue
 			}
 			ne.paths[binary] = binPath
 			break
 		}
-		// TODO: error, so that the kubelet can stop if the paths don't exist
-		// (don't forget that systemd-run is optional)
+		// systemd-run is optional, bailout if we don't find any of the other binaries
+		if ne.paths[binary] == "" && binary != "systemd-run" {
+			return fmt.Errorf("unable to find %v", binary)
+		}
 	}
-	return ne
+	return nil
 }
 
 // Exec executes nsenter commands in hostProcMountNsPath mount namespace
 func (ne *Nsenter) Exec(cmd string, args []string) exec.Cmd {
+	hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
 	fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
 		append([]string{ne.AbsHostPath(cmd)}, args...)...)
 	glog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
-	exec := exec.New()
-	return exec.Command(nsenterPath, fullArgs...)
+	return ne.executor.Command(nsenterPath, fullArgs...)
 }
 
 // AbsHostPath returns the absolute runnable path for a specified command
@@ -119,6 +142,95 @@ func (ne *Nsenter) AbsHostPath(command string) string {
 
 // SupportsSystemd checks whether command systemd-run exists
 func (ne *Nsenter) SupportsSystemd() (string, bool) {
-	systemdRunPath, hasSystemd := ne.paths["systemd-run"]
-	return systemdRunPath, hasSystemd
+	systemdRunPath, ok := ne.paths["systemd-run"]
+	return systemdRunPath, ok && systemdRunPath != ""
 }
+
+// EvalSymlinks returns the path name on the host after evaluating symlinks on the
+// host.
+// mustExist makes EvalSymlinks to return error when the path does not
+// exist. When it's false, it evaluates symlinks of the existing part and
+// blindly adds the non-existing part:
+// pathname: /mnt/volume/non/existing/directory
+//     /mnt/volume exists
+//                non/existing/directory does not exist
+// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
+//    /mnt/foo/non/existing/directory.
+//
+// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
+// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
+func (ne *Nsenter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
+	var args []string
+	if mustExist {
+		// "realpath -e: all components of the path must exist"
+		args = []string{"-e", pathname}
+	} else {
+		// "realpath -m: no path components need exist or be a directory"
+		args = []string{"-m", pathname}
+	}
+	outBytes, err := ne.Exec("realpath", args).CombinedOutput()
+	if err != nil {
+		glog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
+		return "", err
+	}
+	return strings.TrimSpace(string(outBytes)), nil
+}
+
+// KubeletPath returns the path name that can be accessed by containerized
+// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
+// before calling this function
+func (ne *Nsenter) KubeletPath(pathname string) string {
+	return filepath.Join(ne.hostRootFsPath, pathname)
+}
+
+// NewFakeNsenter returns a Nsenter that does not run "nsenter --mount=... --",
+// but runs everything in the same mount namespace as the unit test binary.
+// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
+// This fake Nsenter is enough for most operations, e.g. to resolve symlinks,
+// but it's not enough to call /bin/mount - unit tests don't run as root.
+func NewFakeNsenter(rootfsPath string) (*Nsenter, error) {
+	executor := &fakeExec{
+		rootfsPath: rootfsPath,
+	}
+	// prepare /rootfs/bin, usr/bin and usr/sbin
+	bin := filepath.Join(rootfsPath, "bin")
+	if err := os.Symlink("/bin", bin); err != nil {
+		return nil, err
+	}
+
+	usr := filepath.Join(rootfsPath, "usr")
+	if err := os.Mkdir(usr, 0755); err != nil {
+		return nil, err
+	}
+	usrbin := filepath.Join(usr, "bin")
+	if err := os.Symlink("/usr/bin", usrbin); err != nil {
+		return nil, err
+	}
+	usrsbin := filepath.Join(usr, "sbin")
+	if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
+		return nil, err
+	}
+
+	return NewNsenter(rootfsPath, executor)
+}
+
+type fakeExec struct {
+	rootfsPath string
+}
+
+func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
+	// This will intentionaly panic if Nsenter does not provide enough arguments.
+	realCmd := args[2]
+	realArgs := args[3:]
+	return exec.New().Command(realCmd, realArgs...)
+}
+
+func (fakeExec) LookPath(file string) (string, error) {
+	return "", errors.New("not implemented")
+}
+
+func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
+	return nil
+}
+
+var _ exec.Interface = fakeExec{}

vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter_test.go

@@ -0,0 +1,311 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nsenter
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/utils/exec"
+)
+
+func TestExec(t *testing.T) {
+	tests := []struct {
+		name           string
+		command        string
+		args           []string
+		expectedOutput string
+		expectError    bool
+	}{
+		{
+			name:           "simple command",
+			command:        "echo",
+			args:           []string{"hello", "world"},
+			expectedOutput: "hello world\n",
+		},
+		{
+			name:        "nozero exit code",
+			command:     "false",
+			expectError: true,
+		},
+	}
+
+	executor := fakeExec{
+		rootfsPath: "/rootfs",
+	}
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor:       executor,
+		}
+		cmd := ns.Exec(test.command, test.args)
+		outBytes, err := cmd.CombinedOutput()
+		out := string(outBytes)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if test.expectedOutput != out {
+			t.Errorf("test %q: expected output %q, got %q", test.name, test.expectedOutput, out)
+		}
+	}
+}
+
+func TestKubeletPath(t *testing.T) {
+	tests := []struct {
+		rootfs              string
+		hostpath            string
+		expectedKubeletPath string
+	}{
+		{
+			// simple join
+			"/rootfs",
+			"/some/path",
+			"/rootfs/some/path",
+		},
+		{
+			// squash slashes
+			"/rootfs/",
+			"//some/path",
+			"/rootfs/some/path",
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: test.rootfs,
+		}
+		out := ns.KubeletPath(test.hostpath)
+		if out != test.expectedKubeletPath {
+			t.Errorf("Expected path %q, got %q", test.expectedKubeletPath, out)
+		}
+
+	}
+}
+
+func TestEvalSymlinks(t *testing.T) {
+	tests := []struct {
+		name        string
+		mustExist   bool
+		prepare     func(tmpdir string) (src string, expectedDst string, err error)
+		expectError bool
+	}{
+		{
+			name:      "simple file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				err = ioutil.WriteFile(src, []byte{}, 0644)
+				return src, src, err
+			},
+		},
+		{
+			name:      "non-existing file /src",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "non-existing file /src/ with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "non-existing file /existing/path/src with mustExist=false with existing directories",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				src = filepath.Join(tmpdir, "existing/path")
+				if err := os.MkdirAll(src, 0755); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(src, "src")
+				return src, src, nil
+			},
+		},
+		{
+			name:      "simple symlink /src -> /dst",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = ioutil.WriteFile(dst, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, "", err
+			},
+			expectError: true,
+		},
+		{
+			name:      "dangling symlink /src -> /non-existing-path with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "non-existing-path")
+				src = filepath.Join(tmpdir, "src")
+				err = os.Symlink(dst, src)
+				return src, dst, err
+			},
+		},
+		{
+			name:      "symlink to directory /src/file, where /src is link to /dst",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				if err = os.Mkdir(dst, 0755); err != nil {
+					return "", "", err
+				}
+				dstFile := filepath.Join(dst, "file")
+				if err = ioutil.WriteFile(dstFile, []byte{}, 0644); err != nil {
+					return "", "", err
+				}
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist",
+			mustExist: true,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, "", nil
+			},
+			expectError: true,
+		},
+		{
+			name:      "symlink to non-existing directory: /src/file, where /src is link to /dst and dst does not exist with mustExist=false",
+			mustExist: false,
+			prepare: func(tmpdir string) (src string, expectedDst string, err error) {
+				dst := filepath.Join(tmpdir, "dst")
+				dstFile := filepath.Join(dst, "file")
+
+				src = filepath.Join(tmpdir, "src")
+				if err = os.Symlink(dst, src); err != nil {
+					return "", "", err
+				}
+				srcFile := filepath.Join(src, "file")
+				return srcFile, dstFile, nil
+			},
+		},
+	}
+
+	for _, test := range tests {
+		ns := Nsenter{
+			hostRootFsPath: "/rootfs",
+			executor: fakeExec{
+				rootfsPath: "/rootfs",
+			},
+		}
+
+		tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(tmpdir)
+
+		src, expectedDst, err := test.prepare(tmpdir)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		dst, err := ns.EvalSymlinks(src, test.mustExist)
+		if err != nil && !test.expectError {
+			t.Errorf("Test %q: unexpected error: %s", test.name, err)
+		}
+		if err == nil && test.expectError {
+			t.Errorf("Test %q: expected error, got none", test.name)
+		}
+		if dst != expectedDst {
+			t.Errorf("Test %q: expected destination %q, got %q", test.name, expectedDst, dst)
+		}
+	}
+}
+
+func TestNewNsenter(t *testing.T) {
+	// Create a symlink /tmp/xyz/rootfs -> / and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.Symlink("/", rootfs); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err != nil {
+		t.Errorf("Error: %s", err)
+	}
+}
+
+func TestNewNsenterError(t *testing.T) {
+	// Create empty dir /tmp/xyz/rootfs and use it as rootfs path
+	// It should resolve all binaries correctly, the test runs on Linux
+
+	tmpdir, err := ioutil.TempDir("", "nsenter-hostpath-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	rootfs := filepath.Join(tmpdir, "rootfs")
+	if err = os.MkdirAll(rootfs, 0755); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = NewNsenter(rootfs, exec.New())
+	if err == nil {
+		t.Errorf("Expected error, got none")
+	}
+}

vendor/k8s.io/kubernetes/pkg/util/nsenter/nsenter_unsupported.go

@@ -22,6 +22,12 @@ import (
 	"k8s.io/utils/exec"
 )
 
+const (
+	// DefaultHostRootFsPath is path to host's filesystem mounted into container
+	// with kubelet.
+	DefaultHostRootFsPath = "/rootfs"
+)
+
 // Nsenter is part of experimental support for running the kubelet
 // in a container.
 type Nsenter struct {
@@ -30,8 +36,8 @@ type Nsenter struct {
 }
 
 // NewNsenter constructs a new instance of Nsenter
-func NewNsenter() *Nsenter {
-	return &Nsenter{}
+func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
+	return &Nsenter{}, nil
 }
 
 // Exec executes nsenter commands in hostProcMountNsPath mount namespace

vendor/k8s.io/kubernetes/pkg/util/pod/BUILD

@@ -0,0 +1,39 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["pod.go"],
+    importpath = "k8s.io/kubernetes/pkg/util/pod",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/strategicpatch:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["pod_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/pkg/util/pod/pod.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pod
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+// PatchPodStatus patches pod status.
+func PatchPodStatus(c clientset.Interface, namespace, name string, oldPodStatus, newPodStatus v1.PodStatus) (*v1.Pod, []byte, error) {
+	patchBytes, err := preparePatchBytesforPodStatus(namespace, name, oldPodStatus, newPodStatus)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	updatedPod, err := c.CoreV1().Pods(namespace).Patch(name, types.StrategicMergePatchType, patchBytes, "status")
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to patch status %q for pod %q/%q: %v", patchBytes, namespace, name, err)
+	}
+	return updatedPod, patchBytes, nil
+}
+
+func preparePatchBytesforPodStatus(namespace, name string, oldPodStatus, newPodStatus v1.PodStatus) ([]byte, error) {
+	oldData, err := json.Marshal(v1.Pod{
+		Status: oldPodStatus,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to Marshal oldData for pod %q/%q: %v", namespace, name, err)
+	}
+
+	newData, err := json.Marshal(v1.Pod{
+		Status: newPodStatus,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to Marshal newData for pod %q/%q: %v", namespace, name, err)
+	}
+
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Pod{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to CreateTwoWayMergePatch for pod %q/%q: %v", namespace, name, err)
+	}
+	return patchBytes, nil
+}

vendor/k8s.io/kubernetes/pkg/util/pod/pod_test.go

@@ -0,0 +1,116 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pod
+
+import (
+	"testing"
+
+	"fmt"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/fake"
+	"reflect"
+)
+
+func TestPatchPodStatus(t *testing.T) {
+	ns := "ns"
+	name := "name"
+	client := &fake.Clientset{}
+	client.CoreV1().Pods(ns).Create(&v1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: ns,
+			Name:      name,
+		},
+	})
+
+	testCases := []struct {
+		description        string
+		mutate             func(input v1.PodStatus) v1.PodStatus
+		expectedPatchBytes []byte
+	}{
+		{
+			"no change",
+			func(input v1.PodStatus) v1.PodStatus { return input },
+			[]byte(fmt.Sprintf(`{}`)),
+		},
+		{
+			"message change",
+			func(input v1.PodStatus) v1.PodStatus {
+				input.Message = "random message"
+				return input
+			},
+			[]byte(fmt.Sprintf(`{"status":{"message":"random message"}}`)),
+		},
+		{
+			"pod condition change",
+			func(input v1.PodStatus) v1.PodStatus {
+				input.Conditions[0].Status = v1.ConditionFalse
+				return input
+			},
+			[]byte(fmt.Sprintf(`{"status":{"$setElementOrder/conditions":[{"type":"Ready"},{"type":"PodScheduled"}],"conditions":[{"status":"False","type":"Ready"}]}}`)),
+		},
+		{
+			"additional init container condition",
+			func(input v1.PodStatus) v1.PodStatus {
+				input.InitContainerStatuses = []v1.ContainerStatus{
+					{
+						Name:  "init-container",
+						Ready: true,
+					},
+				}
+				return input
+			},
+			[]byte(fmt.Sprintf(`{"status":{"initContainerStatuses":[{"image":"","imageID":"","lastState":{},"name":"init-container","ready":true,"restartCount":0,"state":{}}]}}`)),
+		},
+	}
+	for _, tc := range testCases {
+		_, patchBytes, err := PatchPodStatus(client, ns, name, getPodStatus(), tc.mutate(getPodStatus()))
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+		}
+		if !reflect.DeepEqual(patchBytes, tc.expectedPatchBytes) {
+			t.Errorf("for test case %q, expect patchBytes: %q, got: %q\n", tc.description, tc.expectedPatchBytes, patchBytes)
+		}
+	}
+}
+
+func getPodStatus() v1.PodStatus {
+	return v1.PodStatus{
+		Phase: v1.PodRunning,
+		Conditions: []v1.PodCondition{
+			{
+				Type:   v1.PodReady,
+				Status: v1.ConditionTrue,
+			},
+			{
+				Type:   v1.PodScheduled,
+				Status: v1.ConditionTrue,
+			},
+		},
+		ContainerStatuses: []v1.ContainerStatus{
+			{
+				Name:  "container1",
+				Ready: true,
+			},
+			{
+				Name:  "container2",
+				Ready: true,
+			},
+		},
+		Message: "Message",
+	}
+}

vendor/k8s.io/kubernetes/pkg/util/pointer/pointer.go

@@ -48,8 +48,12 @@ func AllPtrFieldsNil(obj interface{}) bool {
 
 // Int32Ptr returns a pointer to an int32
 func Int32Ptr(i int32) *int32 {
-	o := i
-	return &o
+	return &i
+}
+
+// Int64Ptr returns a pointer to an int64
+func Int64Ptr(i int64) *int64 {
+	return &i
 }
 
 // Int32PtrDerefOr dereference the int32 ptr and returns it i not nil,
@@ -63,6 +67,5 @@ func Int32PtrDerefOr(ptr *int32, def int32) int32 {
 
 // BoolPtr returns a pointer to a bool
 func BoolPtr(b bool) *bool {
-	o := b
-	return &o
+	return &b
 }

vendor/k8s.io/kubernetes/pkg/util/removeall/removeall_test.go

@@ -75,8 +75,36 @@ func (mounter *fakeMounter) MakeFile(pathname string) error {
 	return nil
 }
 
-func (mounter *fakeMounter) ExistsPath(pathname string) bool {
-	return true
+func (mounter *fakeMounter) ExistsPath(pathname string) (bool, error) {
+	return true, errors.New("not implemented")
+}
+
+func (mounter *fakeMounter) PrepareSafeSubpath(subPath mount.Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return "", nil, nil
+}
+
+func (mounter *fakeMounter) CleanSubPaths(_, _ string) error {
+	return nil
+}
+
+func (mounter *fakeMounter) SafeMakeDir(_, _ string, _ os.FileMode) error {
+	return nil
+}
+
+func (mounter *fakeMounter) GetMountRefs(pathname string) ([]string, error) {
+	return nil, errors.New("not implemented")
+}
+
+func (mounter *fakeMounter) GetFSGroup(pathname string) (int64, error) {
+	return -1, errors.New("not implemented")
+}
+
+func (mounter *fakeMounter) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("not implemented")
+}
+
+func (mounter *fakeMounter) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
 }
 
 func (mounter *fakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {

vendor/k8s.io/kubernetes/pkg/util/verify-util-pkg.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Copyright 2017 The Kubernetes Authors.
 #
@@ -41,7 +41,7 @@ pushd "${BASH_DIR}" > /dev/null
   done
 popd > /dev/null
 
-if [[ ${ret} > 0 ]]; then
+if [[ ${ret} -gt 0 ]]; then
   exit ${ret}
 fi