Update to kube v1.17

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

vendor/k8s.io/kubernetes/pkg/volume/util/atomic_writer.go

@@ -304,7 +304,7 @@ func shouldWriteFile(path string, content []byte) (bool, error) {
 		return false, err
 	}
 
-	return (bytes.Compare(content, contentOnFs) != 0), nil
+	return !bytes.Equal(content, contentOnFs), nil
 }
 
 // pathsToRemove walks the current version of the data directory and

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go

@@ -27,7 +27,7 @@ import (
 	"golang.org/x/sys/unix"
 
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/kubernetes/pkg/volume/util/quota"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota"
 )
 
 // FSInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
@@ -60,7 +60,7 @@ func DiskUsage(path string) (*resource.Quantity, error) {
 	// First check whether the quota system knows about this directory
 	// A nil quantity with no error means that the path does not support quotas
 	// and we should use other mechanisms.
-	data, err := quota.GetConsumption(path)
+	data, err := fsquota.GetConsumption(path)
 	if data != nil {
 		return data, nil
 	} else if err != nil {
@@ -68,9 +68,9 @@ func DiskUsage(path string) (*resource.Quantity, error) {
 	}
 	// Uses the same niceness level as cadvisor.fs does when running du
 	// Uses -B 1 to always scale to a blocksize of 1 byte
-	out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", path).CombinedOutput()
+	out, err := exec.Command("nice", "-n", "19", "du", "-x", "-s", "-B", "1", path).CombinedOutput()
 	if err != nil {
-		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -s -B 1) on path %s with error %v", path, err)
+		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -x -s -B 1) on path %s with error %v", path, err)
 	}
 	used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
 	if err != nil {
@@ -89,7 +89,7 @@ func Find(path string) (int64, error) {
 	// First check whether the quota system knows about this directory
 	// A nil quantity with no error means that the path does not support quotas
 	// and we should use other mechanisms.
-	inodes, err := quota.GetInodes(path)
+	inodes, err := fsquota.GetInodes(path)
 	if inodes != nil {
 		return inodes.Value(), nil
 	} else if err != nil {

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go

@@ -20,6 +20,7 @@ package fs
 
 import (
 	"fmt"
+	"os"
 	"syscall"
 	"unsafe"
 
@@ -58,7 +59,12 @@ func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
 
 // DiskUsage gets disk usage of specified path.
 func DiskUsage(path string) (*resource.Quantity, error) {
-	_, _, usage, _, _, _, err := FsInfo(path)
+	info, err := os.Lstat(path)
+	if err != nil {
+		return nil, err
+	}
+
+	usage, err := diskUsage(path, info)
 	if err != nil {
 		return nil, err
 	}
@@ -75,3 +81,41 @@ func DiskUsage(path string) (*resource.Quantity, error) {
 func Find(path string) (int64, error) {
 	return 0, nil
 }
+
+func diskUsage(currPath string, info os.FileInfo) (int64, error) {
+	var size int64
+
+	if info.Mode()&os.ModeSymlink != 0 {
+		return size, nil
+	}
+
+	size += info.Size()
+
+	if !info.IsDir() {
+		return size, nil
+	}
+
+	dir, err := os.Open(currPath)
+	if err != nil {
+		return size, err
+	}
+	defer dir.Close()
+
+	files, err := dir.Readdir(-1)
+	if err != nil {
+		return size, err
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			s, err := diskUsage(fmt.Sprintf("%s/%s", currPath, file.Name()), file)
+			if err != nil {
+				return size, err
+			}
+			size += s
+		} else {
+			size += file.Size()
+		}
+	}
+	return size, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/project.go

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package quota
+package fsquota
 
 import (
 	"bufio"
@@ -29,7 +29,7 @@ import (
 	"sync"
 
 	"golang.org/x/sys/unix"
-	"k8s.io/kubernetes/pkg/volume/util/quota/common"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota/common"
 )
 
 var projectsFile = "/etc/projects"

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota.go

@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package quota
+package fsquota
 
 import (
+	"k8s.io/utils/mount"
+
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/kubernetes/pkg/features"
-	"k8s.io/kubernetes/pkg/util/mount"
 )
 
 // Interface -- quota interface
@@ -29,7 +31,7 @@ type Interface interface {
 	SupportsQuotas(m mount.Interface, path string) (bool, error)
 	// Assign a quota (picked by the quota mechanism) to a path,
 	// and return it.
-	AssignQuota(m mount.Interface, path string, poduid string, bytes *resource.Quantity) error
+	AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error
 
 	// Get the quota-based storage consumption for the path
 	GetConsumption(path string) (*resource.Quantity, error)
@@ -40,7 +42,7 @@ type Interface interface {
 	// Remove the quota from a path
 	// Implementations may assume that any data covered by the
 	// quota has already been removed.
-	ClearQuota(m mount.Interface, path string, poduid string) error
+	ClearQuota(m mount.Interface, path string) error
 }
 
 func enabledQuotasForMonitoring() bool {

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_linux.go

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package quota
+package fsquota
 
 import (
 	"bufio"
@@ -25,24 +25,26 @@ import (
 	"path/filepath"
 	"sync"
 
-	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/klog"
-	"k8s.io/kubernetes/pkg/util/mount"
-	"k8s.io/kubernetes/pkg/volume/util/quota/common"
+	"k8s.io/utils/mount"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/volume/util/fsquota/common"
 )
 
 // Pod -> ID
-var podQuotaMap = make(map[string]common.QuotaID)
+var podQuotaMap = make(map[types.UID]common.QuotaID)
 
 // Dir -> ID (for convenience)
 var dirQuotaMap = make(map[string]common.QuotaID)
 
 // ID -> pod
-var quotaPodMap = make(map[common.QuotaID]string)
+var quotaPodMap = make(map[common.QuotaID]types.UID)
 
 // Directory -> pod
-var dirPodMap = make(map[string]string)
+var dirPodMap = make(map[string]types.UID)
 
 // Backing device -> applier
 // This is *not* cleaned up; its size will be bounded.
@@ -53,7 +55,7 @@ var dirApplierMap = make(map[string]common.LinuxVolumeQuotaApplier)
 var dirApplierLock sync.RWMutex
 
 // Pod -> refcount
-var podDirCountMap = make(map[string]int)
+var podDirCountMap = make(map[types.UID]int)
 
 // ID -> size
 var quotaSizeMap = make(map[common.QuotaID]int64)
@@ -109,7 +111,7 @@ func clearBackingDev(path string) {
 // Breaking this up helps with testing
 func detectMountpointInternal(m mount.Interface, path string) (string, error) {
 	for path != "" && path != "/" {
-		// per pkg/util/mount/mount_linux this detects all but
+		// per k8s.io/utils/mount/mount_linux this detects all but
 		// a bind mount from one part of a mount to another.
 		// For our purposes that's fine; we simply want the "true"
 		// mount point
@@ -296,7 +298,7 @@ func SupportsQuotas(m mount.Interface, path string) (bool, error) {
 // AssignQuota chooses the quota ID based on the pod UID and path.
 // If the pod UID is identical to another one known, it may (but presently
 // doesn't) choose the same quota ID as other volumes in the pod.
-func AssignQuota(m mount.Interface, path string, poduid string, bytes *resource.Quantity) error {
+func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error {
 	if bytes == nil {
 		return fmt.Errorf("Attempting to assign null quota to %s", path)
 	}
@@ -311,7 +313,7 @@ func AssignQuota(m mount.Interface, path string, poduid string, bytes *resource.
 	// volumes in a pod, we can simply remove this line of code.
 	// If and when we decide permanently that we're going to adop
 	// one quota per volume, we can rip all of the pod code out.
-	poduid = string(uuid.NewUUID())
+	poduid = types.UID(uuid.NewUUID())
 	if pod, ok := dirPodMap[path]; ok && pod != poduid {
 		return fmt.Errorf("Requesting quota on existing directory %s but different pod %s %s", path, pod, poduid)
 	}

vendor/k8s.io/kubernetes/pkg/volume/util/fsquota/quota_unsupported.go

@@ -16,12 +16,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package quota
+package fsquota
 
 import (
 	"errors"
+
+	"k8s.io/utils/mount"
+
 	"k8s.io/apimachinery/pkg/api/resource"
-	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // Dummy quota implementation for systems that do not implement support
@@ -35,7 +38,7 @@ func SupportsQuotas(_ mount.Interface, _ string) (bool, error) {
 }
 
 // AssignQuota -- dummy implementation
-func AssignQuota(_ mount.Interface, _ string, _ string, _ *resource.Quantity) error {
+func AssignQuota(_ mount.Interface, _ string, _ types.UID, _ *resource.Quantity) error {
 	return errNotImplemented
 }
 

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/fake_hostutil.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"errors"
+	"os"
+	"sync"
+
+	"k8s.io/utils/mount"
+)
+
+// FakeHostUtil is a fake HostUtils implementation for testing
+type FakeHostUtil struct {
+	MountPoints []mount.MountPoint
+	Filesystem  map[string]FileType
+
+	mutex sync.Mutex
+}
+
+// NewFakeHostUtil returns a struct that implements the HostUtils interface
+// for testing
+// TODO: no callers were initializing the struct with any MountPoints. Check
+// if those are still being used by any callers and if MountPoints still need
+// to be a part of the struct.
+func NewFakeHostUtil(fs map[string]FileType) *FakeHostUtil {
+	return &FakeHostUtil{
+		Filesystem: fs,
+	}
+}
+
+// Compile-time check to make sure FakeHostUtil implements interface
+var _ HostUtils = &FakeHostUtil{}
+
+// DeviceOpened checks if block device referenced by pathname is in use by
+// checking if is listed as a device in the in-memory mountpoint table.
+func (hu *FakeHostUtil) DeviceOpened(pathname string) (bool, error) {
+	hu.mutex.Lock()
+	defer hu.mutex.Unlock()
+
+	for _, mp := range hu.MountPoints {
+		if mp.Device == pathname {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// PathIsDevice always returns true
+func (hu *FakeHostUtil) PathIsDevice(pathname string) (bool, error) {
+	return true, nil
+}
+
+// GetDeviceNameFromMount given a mount point, find the volume id
+func (hu *FakeHostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// MakeRShared checks if path is shared and bind-mounts it as rshared if needed.
+// No-op for testing
+func (hu *FakeHostUtil) MakeRShared(path string) error {
+	return nil
+}
+
+// GetFileType checks for file/directory/socket/block/character devices.
+// Defaults to Directory if otherwise unspecified.
+func (hu *FakeHostUtil) GetFileType(pathname string) (FileType, error) {
+	if t, ok := hu.Filesystem[pathname]; ok {
+		return t, nil
+	}
+	return FileType("Directory"), nil
+}
+
+// PathExists checks if pathname exists.
+func (hu *FakeHostUtil) PathExists(pathname string) (bool, error) {
+	if _, ok := hu.Filesystem[pathname]; ok {
+		return true, nil
+	}
+	return false, nil
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks.
+// No-op for testing
+func (hu *FakeHostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return pathname, nil
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+// Not implemented for testing
+func (hu *FakeHostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, errors.New("GetOwner not implemented")
+}
+
+// GetSELinuxSupport tests if pathname is on a mount that supports SELinux.
+// Not implemented for testing
+func (hu *FakeHostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errors.New("GetSELinuxSupport not implemented")
+}
+
+// GetMode returns permissions of pathname.
+// Not implemented for testing
+func (hu *FakeHostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errors.New("not implemented")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+
+	"k8s.io/utils/mount"
+)
+
+// FileType enumerates the known set of possible file types.
+type FileType string
+
+const (
+	// FileTypeBlockDev defines a constant for the block device FileType.
+	FileTypeBlockDev FileType = "BlockDevice"
+	// FileTypeCharDev defines a constant for the character device FileType.
+	FileTypeCharDev FileType = "CharDevice"
+	// FileTypeDirectory defines a constant for the directory FileType.
+	FileTypeDirectory FileType = "Directory"
+	// FileTypeFile defines a constant for the file FileType.
+	FileTypeFile FileType = "File"
+	// FileTypeSocket defines a constant for the socket FileType.
+	FileTypeSocket FileType = "Socket"
+	// FileTypeUnknown defines a constant for an unknown FileType.
+	FileTypeUnknown FileType = ""
+)
+
+// HostUtils defines the set of methods for interacting with paths on a host.
+type HostUtils interface {
+	// DeviceOpened determines if the device (e.g. /dev/sdc) is in use elsewhere
+	// on the system, i.e. still mounted.
+	DeviceOpened(pathname string) (bool, error)
+	// PathIsDevice determines if a path is a device.
+	PathIsDevice(pathname string) (bool, error)
+	// GetDeviceNameFromMount finds the device name by checking the mount path
+	// to get the global mount path within its plugin directory.
+	GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error)
+	// MakeRShared checks that given path is on a mount with 'rshared' mount
+	// propagation. If not, it bind-mounts the path as rshared.
+	MakeRShared(path string) error
+	// GetFileType checks for file/directory/socket/block/character devices.
+	GetFileType(pathname string) (FileType, error)
+	// PathExists tests if the given path already exists
+	// Error is returned on any other error than "file not found".
+	PathExists(pathname string) (bool, error)
+	// EvalHostSymlinks returns the path name after evaluating symlinks.
+	EvalHostSymlinks(pathname string) (string, error)
+	// GetOwner returns the integer ID for the user and group of the given path
+	GetOwner(pathname string) (int64, int64, error)
+	// GetSELinuxSupport returns true if given path is on a mount that supports
+	// SELinux.
+	GetSELinuxSupport(pathname string) (bool, error)
+	// GetMode returns permissions of the path.
+	GetMode(pathname string) (os.FileMode, error)
+}
+
+// Compile-time check to ensure all HostUtil implementations satisfy
+// the Interface.
+var _ HostUtils = &HostUtil{}
+
+// getFileType checks for file/directory/socket and block/character devices.
+func getFileType(pathname string) (FileType, error) {
+	var pathType FileType
+	info, err := os.Stat(pathname)
+	if os.IsNotExist(err) {
+		return pathType, fmt.Errorf("path %q does not exist", pathname)
+	}
+	// err in call to os.Stat
+	if err != nil {
+		return pathType, err
+	}
+
+	// checks whether the mode is the target mode.
+	isSpecificMode := func(mode, targetMode os.FileMode) bool {
+		return mode&targetMode == targetMode
+	}
+
+	mode := info.Mode()
+	if mode.IsDir() {
+		return FileTypeDirectory, nil
+	} else if mode.IsRegular() {
+		return FileTypeFile, nil
+	} else if isSpecificMode(mode, os.ModeSocket) {
+		return FileTypeSocket, nil
+	} else if isSpecificMode(mode, os.ModeDevice) {
+		if isSpecificMode(mode, os.ModeCharDevice) {
+			return FileTypeCharDev, nil
+		}
+		return FileTypeBlockDev, nil
+	}
+
+	return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_linux.go

@@ -0,0 +1,286 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+	"k8s.io/klog"
+	"k8s.io/utils/mount"
+	utilpath "k8s.io/utils/path"
+)
+
+const (
+	// Location of the mountinfo file
+	procMountInfoPath = "/proc/self/mountinfo"
+)
+
+// HostUtil implements HostUtils for Linux platforms.
+type HostUtil struct {
+}
+
+// NewHostUtil returns a struct that implements the HostUtils interface on
+// linux platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
+// If pathname is not a device, log and return false with nil error.
+// If open returns errno EBUSY, return true with nil error.
+// If open returns nil, return false with nil error.
+// Otherwise, return false with error
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return ExclusiveOpenFailsOnDevice(pathname)
+}
+
+// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
+// to a device.
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	pathType, err := hu.GetFileType(pathname)
+	isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
+	return isDevice, err
+}
+
+// ExclusiveOpenFailsOnDevice is shared with NsEnterMounter
+func ExclusiveOpenFailsOnDevice(pathname string) (bool, error) {
+	var isDevice bool
+	finfo, err := os.Stat(pathname)
+	if os.IsNotExist(err) {
+		isDevice = false
+	}
+	// err in call to os.Stat
+	if err != nil {
+		return false, fmt.Errorf(
+			"PathIsDevice failed for path %q: %v",
+			pathname,
+			err)
+	}
+	// path refers to a device
+	if finfo.Mode()&os.ModeDevice != 0 {
+		isDevice = true
+	}
+
+	if !isDevice {
+		klog.Errorf("Path %q is not referring to a device.", pathname)
+		return false, nil
+	}
+	fd, errno := unix.Open(pathname, unix.O_RDONLY|unix.O_EXCL|unix.O_CLOEXEC, 0)
+	// If the device is in use, open will return an invalid fd.
+	// When this happens, it is expected that Close will fail and throw an error.
+	defer unix.Close(fd)
+	if errno == nil {
+		// device not in use
+		return false, nil
+	} else if errno == unix.EBUSY {
+		// device is in use
+		return true, nil
+	}
+	// error during call to Open
+	return false, errno
+}
+
+// GetDeviceNameFromMount given a mount point, find the device name from its global mount point
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// getDeviceNameFromMountLinux find the device name from /proc/mounts in which
+// the mount path reference should match the given plugin mount directory. In case no mount path reference
+// matches, returns the volume name taken from its given mountPath
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	refs, err := mounter.GetMountRefs(mountPath)
+	if err != nil {
+		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
+		return "", err
+	}
+	if len(refs) == 0 {
+		klog.V(4).Infof("Directory %s is not mounted", mountPath)
+		return "", fmt.Errorf("directory %s is not mounted", mountPath)
+	}
+	for _, ref := range refs {
+		if strings.HasPrefix(ref, pluginMountDir) {
+			volumeID, err := filepath.Rel(pluginMountDir, ref)
+			if err != nil {
+				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
+				return "", err
+			}
+			return volumeID, nil
+		}
+	}
+
+	return path.Base(mountPath), nil
+}
+
+// MakeRShared checks that given path is on a mount with 'rshared' mount
+// propagation. If not, it bind-mounts the path as rshared.
+func (hu *HostUtil) MakeRShared(path string) error {
+	return DoMakeRShared(path, procMountInfoPath)
+}
+
+// GetFileType checks for file/directory/socket/block/character devices.
+func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
+	return getFileType(pathname)
+}
+
+// PathExists tests if the given path already exists
+// Error is returned on any other error than "file not found".
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks.
+// TODO once the nsenter implementation is removed, this method can be removed
+// from the interface and filepath.EvalSymlinks used directly
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return filepath.EvalSymlinks(pathname)
+}
+
+// isShared returns true, if given path is on a mount point that has shared
+// mount propagation.
+func isShared(mount string, mountInfoPath string) (bool, error) {
+	info, err := findMountInfo(mount, mountInfoPath)
+	if err != nil {
+		return false, err
+	}
+
+	// parse optional parameters
+	for _, opt := range info.OptionalFields {
+		if strings.HasPrefix(opt, "shared:") {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func findMountInfo(path, mountInfoPath string) (mount.MountInfo, error) {
+	infos, err := mount.ParseMountInfo(mountInfoPath)
+	if err != nil {
+		return mount.MountInfo{}, err
+	}
+
+	// process /proc/xxx/mountinfo in backward order and find the first mount
+	// point that is prefix of 'path' - that's the mount where path resides
+	var info *mount.MountInfo
+	for i := len(infos) - 1; i >= 0; i-- {
+		if mount.PathWithinBase(path, infos[i].MountPoint) {
+			info = &infos[i]
+			break
+		}
+	}
+	if info == nil {
+		return mount.MountInfo{}, fmt.Errorf("cannot find mount point for %q", path)
+	}
+	return *info, nil
+}
+
+// DoMakeRShared is common implementation of MakeRShared on Linux. It checks if
+// path is shared and bind-mounts it as rshared if needed. mountCmd and
+// mountArgs are expected to contain mount-like command, DoMakeRShared will add
+// '--bind <path> <path>' and '--make-rshared <path>' to mountArgs.
+func DoMakeRShared(path string, mountInfoFilename string) error {
+	shared, err := isShared(path, mountInfoFilename)
+	if err != nil {
+		return err
+	}
+	if shared {
+		klog.V(4).Infof("Directory %s is already on a shared mount", path)
+		return nil
+	}
+
+	klog.V(2).Infof("Bind-mounting %q with shared mount propagation", path)
+	// mount --bind /var/lib/kubelet /var/lib/kubelet
+	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_BIND, "" /*data*/); err != nil {
+		return fmt.Errorf("failed to bind-mount %s: %v", path, err)
+	}
+
+	// mount --make-rshared /var/lib/kubelet
+	if err := syscall.Mount(path, path, "" /*fstype*/, syscall.MS_SHARED|syscall.MS_REC, "" /*data*/); err != nil {
+		return fmt.Errorf("failed to make %s rshared: %v", path, err)
+	}
+
+	return nil
+}
+
+// GetSELinux is common implementation of GetSELinuxSupport on Linux.
+func GetSELinux(path string, mountInfoFilename string) (bool, error) {
+	info, err := findMountInfo(path, mountInfoFilename)
+	if err != nil {
+		return false, err
+	}
+
+	// "seclabel" can be both in mount options and super options.
+	for _, opt := range info.SuperOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	for _, opt := range info.MountOptions {
+		if opt == "seclabel" {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// GetSELinuxSupport returns true if given path is on a mount that supports
+// SELinux.
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return GetSELinux(pathname, procMountInfoPath)
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	realpath, err := filepath.EvalSymlinks(pathname)
+	if err != nil {
+		return -1, -1, err
+	}
+	return GetOwnerLinux(realpath)
+}
+
+// GetMode returns permissions of the path.
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return GetModeLinux(pathname)
+}
+
+// GetOwnerLinux is shared between Linux and NsEnterMounter
+// pathname must already be evaluated for symlinks
+func GetOwnerLinux(pathname string) (int64, int64, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return -1, -1, err
+	}
+	stat := info.Sys().(*syscall.Stat_t)
+	return int64(stat.Uid), int64(stat.Gid), nil
+}
+
+// GetModeLinux is shared between Linux and NsEnterMounter
+func GetModeLinux(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_unsupported.go

@@ -0,0 +1,102 @@
+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"errors"
+	"os"
+
+	"k8s.io/utils/mount"
+)
+
+// HostUtil is an HostUtils implementation that allows compilation on
+// unsupported platforms
+type HostUtil struct{}
+
+// NewHostUtil returns a struct that implements the HostUtils interface on
+// unsupported platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+var errUnsupported = errors.New("volume/util/hostutil on this platform is not supported")
+
+// DeviceOpened always returns an error on unsupported platforms
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return false, errUnsupported
+}
+
+// PathIsDevice always returns an error on unsupported platforms
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	return true, errUnsupported
+}
+
+// GetDeviceNameFromMount always returns an error on unsupported platforms
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// MakeRShared always returns an error on unsupported platforms
+func (hu *HostUtil) MakeRShared(path string) error {
+	return errUnsupported
+}
+
+// GetFileType always returns an error on unsupported platforms
+func (hu *HostUtil) GetFileType(pathname string) (FileType, error) {
+	return FileType("fake"), errUnsupported
+}
+
+// MakeFile always returns an error on unsupported platforms
+func (hu *HostUtil) MakeFile(pathname string) error {
+	return errUnsupported
+}
+
+// MakeDir always returns an error on unsupported platforms
+func (hu *HostUtil) MakeDir(pathname string) error {
+	return errUnsupported
+}
+
+// PathExists always returns an error on unsupported platforms
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return true, errUnsupported
+}
+
+// EvalHostSymlinks always returns an error on unsupported platforms
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return "", errUnsupported
+}
+
+// GetOwner always returns an error on unsupported platforms
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, errUnsupported
+}
+
+// GetSELinuxSupport always returns an error on unsupported platforms
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, errUnsupported
+}
+
+//GetMode always returns an error on unsupported platforms
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	return 0, errUnsupported
+}
+
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return "", errUnsupported
+}

vendor/k8s.io/kubernetes/pkg/volume/util/hostutil/hostutil_windows.go

@@ -0,0 +1,124 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package hostutil
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/klog"
+	"k8s.io/utils/mount"
+	utilpath "k8s.io/utils/path"
+)
+
+// HostUtil implements HostUtils for Windows platforms.
+type HostUtil struct{}
+
+// NewHostUtil returns a struct that implements HostUtils on Windows platforms
+func NewHostUtil() *HostUtil {
+	return &HostUtil{}
+}
+
+// GetDeviceNameFromMount given a mnt point, find the device
+func (hu *HostUtil) GetDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	return getDeviceNameFromMount(mounter, mountPath, pluginMountDir)
+}
+
+// getDeviceNameFromMount find the device(drive) name in which
+// the mount path reference should match the given plugin mount directory. In case no mount path reference
+// matches, returns the volume name taken from its given mountPath
+func getDeviceNameFromMount(mounter mount.Interface, mountPath, pluginMountDir string) (string, error) {
+	refs, err := mounter.GetMountRefs(mountPath)
+	if err != nil {
+		klog.V(4).Infof("GetMountRefs failed for mount path %q: %v", mountPath, err)
+		return "", err
+	}
+	if len(refs) == 0 {
+		return "", fmt.Errorf("directory %s is not mounted", mountPath)
+	}
+	basemountPath := mount.NormalizeWindowsPath(pluginMountDir)
+	for _, ref := range refs {
+		if strings.Contains(ref, basemountPath) {
+			volumeID, err := filepath.Rel(mount.NormalizeWindowsPath(basemountPath), ref)
+			if err != nil {
+				klog.Errorf("Failed to get volume id from mount %s - %v", mountPath, err)
+				return "", err
+			}
+			return volumeID, nil
+		}
+	}
+
+	return path.Base(mountPath), nil
+}
+
+// DeviceOpened determines if the device is in use elsewhere
+func (hu *HostUtil) DeviceOpened(pathname string) (bool, error) {
+	return false, nil
+}
+
+// PathIsDevice determines if a path is a device.
+func (hu *HostUtil) PathIsDevice(pathname string) (bool, error) {
+	return false, nil
+}
+
+// MakeRShared checks that given path is on a mount with 'rshared' mount
+// propagation. Empty implementation here.
+func (hu *HostUtil) MakeRShared(path string) error {
+	return nil
+}
+
+// GetFileType checks for sockets/block/character devices
+func (hu *(HostUtil)) GetFileType(pathname string) (FileType, error) {
+	return getFileType(pathname)
+}
+
+// PathExists checks whether the path exists
+func (hu *HostUtil) PathExists(pathname string) (bool, error) {
+	return utilpath.Exists(utilpath.CheckFollowSymlink, pathname)
+}
+
+// EvalHostSymlinks returns the path name after evaluating symlinks
+func (hu *HostUtil) EvalHostSymlinks(pathname string) (string, error) {
+	return filepath.EvalSymlinks(pathname)
+}
+
+// GetOwner returns the integer ID for the user and group of the given path
+// Note that on windows, it always returns 0. We actually don't set Group on
+// windows platform, see SetVolumeOwnership implementation.
+func (hu *HostUtil) GetOwner(pathname string) (int64, int64, error) {
+	return -1, -1, nil
+}
+
+// GetSELinuxSupport returns a boolean indicating support for SELinux.
+// Windows does not support SELinux.
+func (hu *HostUtil) GetSELinuxSupport(pathname string) (bool, error) {
+	return false, nil
+}
+
+// GetMode returns permissions of the path.
+func (hu *HostUtil) GetMode(pathname string) (os.FileMode, error) {
+	info, err := os.Stat(pathname)
+	if err != nil {
+		return 0, err
+	}
+	return info.Mode(), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/metrics.go

@@ -20,7 +20,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 	"k8s.io/kubernetes/pkg/volume"
 )
 
@@ -29,36 +30,48 @@ const (
 	statusFailUnknown = "fail-unknown"
 )
 
-var storageOperationMetric = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Name:    "storage_operation_duration_seconds",
-		Help:    "Storage operation duration",
-		Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50, 120, 300, 600},
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var storageOperationMetric = metrics.NewHistogramVec(
+	&metrics.HistogramOpts{
+		Name:           "storage_operation_duration_seconds",
+		Help:           "Storage operation duration",
+		Buckets:        []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50, 120, 300, 600},
+		StabilityLevel: metrics.ALPHA,
 	},
 	[]string{"volume_plugin", "operation_name"},
 )
 
-var storageOperationErrorMetric = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Name: "storage_operation_errors_total",
-		Help: "Storage operation errors",
+var storageOperationErrorMetric = metrics.NewCounterVec(
+	&metrics.CounterOpts{
+		Name:           "storage_operation_errors_total",
+		Help:           "Storage operation errors",
+		StabilityLevel: metrics.ALPHA,
 	},
 	[]string{"volume_plugin", "operation_name"},
 )
 
-var storageOperationStatusMetric = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Name: "storage_operation_status_count",
-		Help: "Storage operation return statuses count",
+var storageOperationStatusMetric = metrics.NewCounterVec(
+	&metrics.CounterOpts{
+		Name:           "storage_operation_status_count",
+		Help:           "Storage operation return statuses count",
+		StabilityLevel: metrics.ALPHA,
 	},
 	[]string{"volume_plugin", "operation_name", "status"},
 )
 
-var storageOperationEndToEndLatencyMetric = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Name:    "volume_operation_total_seconds",
-		Help:    "Storage operation end to end duration in seconds",
-		Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50, 120, 300, 600},
+var storageOperationEndToEndLatencyMetric = metrics.NewHistogramVec(
+	&metrics.HistogramOpts{
+		Name:           "volume_operation_total_seconds",
+		Help:           "Storage operation end to end duration in seconds",
+		Buckets:        []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50, 120, 300, 600},
+		StabilityLevel: metrics.ALPHA,
 	},
 	[]string{"plugin_name", "operation_name"},
 )
@@ -68,10 +81,12 @@ func init() {
 }
 
 func registerMetrics() {
-	prometheus.MustRegister(storageOperationMetric)
-	prometheus.MustRegister(storageOperationErrorMetric)
-	prometheus.MustRegister(storageOperationStatusMetric)
-	prometheus.MustRegister(storageOperationEndToEndLatencyMetric)
+	// legacyregistry is the internal k8s wrapper around the prometheus
+	// global registry, used specifically for metric stability enforcement
+	legacyregistry.MustRegister(storageOperationMetric)
+	legacyregistry.MustRegister(storageOperationErrorMetric)
+	legacyregistry.MustRegister(storageOperationStatusMetric)
+	legacyregistry.MustRegister(storageOperationEndToEndLatencyMetric)
 }
 
 // OperationCompleteHook returns a hook to call when an operation is completed

vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	"k8s.io/api/core/v1"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 )
 
 // getNestedMountpoints returns a list of mountpoint directories that should be created
@@ -70,16 +71,19 @@ func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) {
 		}
 		return nil
 	}
-	for _, container := range pod.Spec.InitContainers {
-		if err := checkContainer(&container); err != nil {
-			return nil, err
-		}
-	}
-	for _, container := range pod.Spec.Containers {
-		if err := checkContainer(&container); err != nil {
-			return nil, err
+
+	var retErr error
+	podutil.VisitContainers(&pod.Spec, func(c *v1.Container) bool {
+		retErr = checkContainer(c)
+		if retErr != nil {
+			return false
 		}
+		return true
+	})
+	if retErr != nil {
+		return nil, retErr
 	}
+
 	return retval, nil
 }
 

vendor/k8s.io/kubernetes/pkg/volume/util/resize_util.go

@@ -20,6 +20,8 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"k8s.io/utils/mount"
+
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -28,7 +30,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/util/resizefs"
 	"k8s.io/kubernetes/pkg/volume"
 	volumetypes "k8s.io/kubernetes/pkg/volume/util/types"

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go

@@ -28,10 +28,8 @@ import (
 	"syscall"
 
 	"golang.org/x/sys/unix"
-
 	"k8s.io/klog"
-
-	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/mount"
 )
 
 const (
@@ -241,6 +239,12 @@ func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string)
 			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
 				return err
 			}
+
+			if info.IsDir() {
+				// skip subdirs of the volume: it only matters the first level to unmount, otherwise it would try to unmount subdir of the volume
+				return filepath.SkipDir
+			}
+
 			return nil
 		})
 		if err != nil {
@@ -398,7 +402,7 @@ func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
 			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
 		}
 		// Dive into the created directory
-		childFD, err = syscall.Openat(parentFD, dir, nofollowFlags, 0)
+		childFD, err = syscall.Openat(parentFD, dir, nofollowFlags|unix.O_CLOEXEC, 0)
 		if err != nil {
 			return fmt.Errorf("cannot open %s: %s", currentPath, err)
 		}
@@ -454,7 +458,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) {
 	// This should be faster than looping through all dirs and calling os.Stat()
 	// on each of them, as the symlinks are resolved only once with OpenAt().
 	currentPath := base
-	fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
+	fd, err := syscall.Open(currentPath, syscall.O_RDONLY|syscall.O_CLOEXEC, 0)
 	if err != nil {
 		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
 	}
@@ -466,7 +470,7 @@ func findExistingPrefix(base, pathname string) (string, []string, error) {
 	for i, dir := range dirs {
 		// Using O_PATH here will prevent hangs in case user replaces directory with
 		// fifo
-		childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
+		childFD, err := syscall.Openat(fd, dir, unix.O_PATH|unix.O_CLOEXEC, 0)
 		if err != nil {
 			if os.IsNotExist(err) {
 				return currentPath, dirs[i:], nil
@@ -499,7 +503,7 @@ func doSafeOpen(pathname string, base string) (int, error) {
 
 	// Assumption: base is the only directory that we have under control.
 	// Base dir is not allowed to be a symlink.
-	parentFD, err := syscall.Open(base, nofollowFlags, 0)
+	parentFD, err := syscall.Open(base, nofollowFlags|unix.O_CLOEXEC, 0)
 	if err != nil {
 		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
 	}
@@ -531,7 +535,7 @@ func doSafeOpen(pathname string, base string) (int, error) {
 		}
 
 		klog.V(5).Infof("Opening path %s", currentPath)
-		childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
+		childFD, err = syscall.Openat(parentFD, seg, openFDFlags|unix.O_CLOEXEC, 0)
 		if err != nil {
 			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
 		}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_nsenter.go

@@ -1,186 +0,0 @@
-// +build linux
-
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package subpath
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-
-	"golang.org/x/sys/unix"
-
-	"k8s.io/klog"
-	"k8s.io/utils/nsenter"
-
-	"k8s.io/kubernetes/pkg/util/mount"
-)
-
-type subpathNSE struct {
-	mounter mount.Interface
-	ne      *nsenter.Nsenter
-	rootDir string
-}
-
-// Compile time-check for all implementers of subpath interface
-var _ Interface = &subpathNSE{}
-
-// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
-// It is only valid on Linux systems
-func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
-	return &subpathNSE{
-		mounter: mounter,
-		ne:      ne,
-		rootDir: rootDir,
-	}
-}
-
-func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
-	return doCleanSubPaths(sp.mounter, podDir, volumeName)
-}
-
-func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
-	// Bind-mount the subpath to avoid using symlinks in subpaths.
-	newHostPath, err = sp.doNsEnterBindSubPath(subPath)
-
-	// There is no action when the container starts. Bind-mount will be cleaned
-	// when container stops by CleanSubPaths.
-	cleanupAction = nil
-	return newHostPath, cleanupAction, err
-}
-
-func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
-	fullSubdirPath := filepath.Join(base, subdir)
-	evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
-	}
-	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
-
-	evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
-	if err != nil {
-		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
-	}
-	evaluatedBase = filepath.Clean(evaluatedBase)
-
-	rootDir := filepath.Clean(sp.rootDir)
-	if mount.PathWithinBase(evaluatedBase, rootDir) {
-		// Base is in /var/lib/kubelet. This directory is shared between the
-		// container with kubelet and the host. We don't need to add '/rootfs'.
-		// This is useful when /rootfs is mounted as read-only - we can still
-		// create subpaths for paths in /var/lib/kubelet.
-		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
-	}
-
-	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
-	// the directory there.
-	// This requires /rootfs to be writable.
-	kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
-	kubeletBase := sp.ne.KubeletPath(evaluatedBase)
-	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
-}
-
-func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
-	// Linux, kubelet runs in a container:
-	// - safely open the subpath
-	// - bind-mount the subpath to target (this can be unsafe)
-	// - check that we mounted the right thing by comparing device ID and inode
-	//   of the subpath (via safely opened fd) and the target (that's under our
-	//   control)
-
-	// Evaluate all symlinks here once for all subsequent functions.
-	evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
-	}
-	evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
-	if err != nil {
-		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
-	}
-	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
-	subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
-	subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
-
-	// Check the subpath is correct and open it
-	fd, err := safeOpenSubPath(sp.mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	defer syscall.Close(fd)
-
-	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
-	if err != nil {
-		return "", err
-	}
-	if alreadyMounted {
-		return bindPathTarget, nil
-	}
-
-	success := false
-	defer func() {
-		// Cleanup subpath on error
-		if !success {
-			klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
-			if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
-				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
-			}
-		}
-	}()
-
-	// Leap of faith: optimistically expect that nobody has modified previously
-	// expanded evalSubPath with evil symlinks and bind-mount it.
-	// Mount is done on the host! don't use kubelet path!
-	klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
-	if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
-		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
-	}
-
-	// Check that the bind-mount target is the same inode and device as the
-	// source that we keept open, i.e. we mounted the right thing.
-	err = checkDeviceInode(fd, bindPathTarget)
-	if err != nil {
-		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
-	}
-
-	success = true
-	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
-	return bindPathTarget, nil
-}
-
-// checkDeviceInode checks that opened file and path represent the same file.
-func checkDeviceInode(fd int, path string) error {
-	var srcStat, dstStat unix.Stat_t
-	err := unix.Fstat(fd, &srcStat)
-	if err != nil {
-		return fmt.Errorf("error running fstat on subpath FD: %v", err)
-	}
-
-	err = unix.Stat(path, &dstStat)
-	if err != nil {
-		return fmt.Errorf("error running fstat on %s: %v", path, err)
-	}
-
-	if srcStat.Dev != dstStat.Dev {
-		return fmt.Errorf("different device number")
-	}
-	if srcStat.Ino != dstStat.Ino {
-		return fmt.Errorf("different inode")
-	}
-	return nil
-}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go

@@ -22,7 +22,7 @@ import (
 	"errors"
 	"os"
 
-	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/mount"
 	"k8s.io/utils/nsenter"
 )
 

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go

@@ -26,7 +26,7 @@ import (
 	"syscall"
 
 	"k8s.io/klog"
-	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/mount"
 	"k8s.io/utils/nsenter"
 )
 

vendor/k8s.io/kubernetes/pkg/volume/util/util.go

@@ -22,27 +22,31 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strings"
 
+	"k8s.io/klog"
+	utilexec "k8s.io/utils/exec"
+	"k8s.io/utils/mount"
+	utilstrings "k8s.io/utils/strings"
+
 	v1 "k8s.io/api/core/v1"
 	storage "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
+	apiruntime "k8s.io/apimachinery/pkg/runtime"
 	utypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	podutil "k8s.io/kubernetes/pkg/api/v1/pod"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 	"k8s.io/kubernetes/pkg/features"
-	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 	"k8s.io/kubernetes/pkg/volume/util/types"
 	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
-	utilstrings "k8s.io/utils/strings"
 )
 
 const (
@@ -195,7 +199,7 @@ func LoadPodFromFile(filePath string) (*v1.Pod, error) {
 	pod := &v1.Pod{}
 
 	codec := legacyscheme.Codecs.UniversalDecoder()
-	if err := runtime.DecodeInto(codec, podDef, pod); err != nil {
+	if err := apiruntime.DecodeInto(codec, podDef, pod); err != nil {
 		return nil, fmt.Errorf("failed decoding file: %v", err)
 	}
 	return pod, nil
@@ -423,14 +427,6 @@ func GetVolumeMode(volumeSpec *volume.Spec) (v1.PersistentVolumeMode, error) {
 	return "", fmt.Errorf("cannot get volumeMode for volume: %v", volumeSpec.Name())
 }
 
-// GetPersistentVolumeClaimVolumeMode retrieves VolumeMode from pvc.
-func GetPersistentVolumeClaimVolumeMode(claim *v1.PersistentVolumeClaim) (v1.PersistentVolumeMode, error) {
-	if claim.Spec.VolumeMode != nil {
-		return *claim.Spec.VolumeMode, nil
-	}
-	return "", fmt.Errorf("cannot get volumeMode from pvc: %v", claim.Name)
-}
-
 // GetPersistentVolumeClaimQualifiedName returns a qualified name for pvc.
 func GetPersistentVolumeClaimQualifiedName(claim *v1.PersistentVolumeClaim) string {
 	return utilstrings.JoinQualifiedName(claim.GetNamespace(), claim.GetName())
@@ -507,30 +503,74 @@ func MakeAbsolutePath(goos, path string) string {
 	return "c:\\" + path
 }
 
-// MapBlockVolume is a utility function to provide a common way of mounting
+// MapBlockVolume is a utility function to provide a common way of mapping
 // block device path for a specified volume and pod.  This function should be
 // called by volume plugins that implements volume.BlockVolumeMapper.Map() method.
 func MapBlockVolume(
+	blkUtil volumepathhandler.BlockVolumePathHandler,
 	devicePath,
 	globalMapPath,
 	podVolumeMapPath,
 	volumeMapName string,
 	podUID utypes.UID,
 ) error {
-	blkUtil := volumepathhandler.NewBlockVolumePathHandler()
-
-	// map devicePath to global node path
-	mapErr := blkUtil.MapDevice(devicePath, globalMapPath, string(podUID))
+	// map devicePath to global node path as bind mount
+	mapErr := blkUtil.MapDevice(devicePath, globalMapPath, string(podUID), true /* bindMount */)
 	if mapErr != nil {
-		return mapErr
+		return fmt.Errorf("blkUtil.MapDevice failed. devicePath: %s, globalMapPath:%s, podUID: %s, bindMount: %v: %v",
+			devicePath, globalMapPath, string(podUID), true, mapErr)
 	}
 
 	// map devicePath to pod volume path
-	mapErr = blkUtil.MapDevice(devicePath, podVolumeMapPath, volumeMapName)
+	mapErr = blkUtil.MapDevice(devicePath, podVolumeMapPath, volumeMapName, false /* bindMount */)
 	if mapErr != nil {
-		return mapErr
+		return fmt.Errorf("blkUtil.MapDevice failed. devicePath: %s, podVolumeMapPath:%s, volumeMapName: %s, bindMount: %v: %v",
+			devicePath, podVolumeMapPath, volumeMapName, false, mapErr)
 	}
 
+	// Take file descriptor lock to keep a block device opened. Otherwise, there is a case
+	// that the block device is silently removed and attached another device with the same name.
+	// Container runtime can't handle this problem. To avoid unexpected condition fd lock
+	// for the block device is required.
+	_, mapErr = blkUtil.AttachFileDevice(filepath.Join(globalMapPath, string(podUID)))
+	if mapErr != nil {
+		return fmt.Errorf("blkUtil.AttachFileDevice failed. globalMapPath:%s, podUID: %s: %v",
+			globalMapPath, string(podUID), mapErr)
+	}
+
+	return nil
+}
+
+// UnmapBlockVolume is a utility function to provide a common way of unmapping
+// block device path for a specified volume and pod.  This function should be
+// called by volume plugins that implements volume.BlockVolumeMapper.Map() method.
+func UnmapBlockVolume(
+	blkUtil volumepathhandler.BlockVolumePathHandler,
+	globalUnmapPath,
+	podDeviceUnmapPath,
+	volumeMapName string,
+	podUID utypes.UID,
+) error {
+	// Release file descriptor lock.
+	err := blkUtil.DetachFileDevice(filepath.Join(globalUnmapPath, string(podUID)))
+	if err != nil {
+		return fmt.Errorf("blkUtil.DetachFileDevice failed. globalUnmapPath:%s, podUID: %s: %v",
+			globalUnmapPath, string(podUID), err)
+	}
+
+	// unmap devicePath from pod volume path
+	unmapDeviceErr := blkUtil.UnmapDevice(podDeviceUnmapPath, volumeMapName, false /* bindMount */)
+	if unmapDeviceErr != nil {
+		return fmt.Errorf("blkUtil.DetachFileDevice failed. podDeviceUnmapPath:%s, volumeMapName: %s, bindMount: %v: %v",
+			podDeviceUnmapPath, volumeMapName, false, unmapDeviceErr)
+	}
+
+	// unmap devicePath from global node path
+	unmapDeviceErr = blkUtil.UnmapDevice(globalUnmapPath, string(podUID), true /* bindMount */)
+	if unmapDeviceErr != nil {
+		return fmt.Errorf("blkUtil.DetachFileDevice failed. globalUnmapPath:%s, podUID: %s, bindMount: %v: %v",
+			globalUnmapPath, string(podUID), true, unmapDeviceErr)
+	}
 	return nil
 }
 
@@ -548,3 +588,59 @@ func IsLocalEphemeralVolume(volume v1.Volume) bool {
 		(volume.EmptyDir != nil && volume.EmptyDir.Medium != v1.StorageMediumMemory) ||
 		volume.ConfigMap != nil || volume.DownwardAPI != nil
 }
+
+// GetPodVolumeNames returns names of volumes that are used in a pod,
+// either as filesystem mount or raw block device.
+func GetPodVolumeNames(pod *v1.Pod) (mounts sets.String, devices sets.String) {
+	mounts = sets.NewString()
+	devices = sets.NewString()
+
+	podutil.VisitContainers(&pod.Spec, func(container *v1.Container) bool {
+		if container.VolumeMounts != nil {
+			for _, mount := range container.VolumeMounts {
+				mounts.Insert(mount.Name)
+			}
+		}
+		// TODO: remove feature gate check after no longer needed
+		if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) &&
+			container.VolumeDevices != nil {
+			for _, device := range container.VolumeDevices {
+				devices.Insert(device.Name)
+			}
+		}
+		return true
+	})
+	return
+}
+
+// HasMountRefs checks if the given mountPath has mountRefs.
+// TODO: this is a workaround for the unmount device issue caused by gci mounter.
+// In GCI cluster, if gci mounter is used for mounting, the container started by mounter
+// script will cause additional mounts created in the container. Since these mounts are
+// irrelevant to the original mounts, they should be not considered when checking the
+// mount references. Current solution is to filter out those mount paths that contain
+// the string of original mount path.
+// Plan to work on better approach to solve this issue.
+func HasMountRefs(mountPath string, mountRefs []string) bool {
+	for _, ref := range mountRefs {
+		if !strings.Contains(ref, mountPath) {
+			return true
+		}
+	}
+	return false
+}
+
+//WriteVolumeCache flush disk data given the spcified mount path
+func WriteVolumeCache(deviceMountPath string, exec utilexec.Interface) error {
+	// If runtime os is windows, execute Write-VolumeCache powershell command on the disk
+	if runtime.GOOS == "windows" {
+		cmd := fmt.Sprintf("Get-Volume -FilePath %s | Write-Volumecache", deviceMountPath)
+		output, err := exec.Command("powershell", "/c", cmd).CombinedOutput()
+		klog.Infof("command (%q) execeuted: %v, output: %q", cmd, err, string(output))
+		if err != nil {
+			return fmt.Errorf("command (%q) failed: %v, output: %q", cmd, err, string(output))
+		}
+	}
+	// For linux runtime, it skips because unmount will automatically flush disk data
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler.go

@@ -23,12 +23,15 @@ import (
 	"path/filepath"
 
 	"k8s.io/klog"
+	utilexec "k8s.io/utils/exec"
+	"k8s.io/utils/mount"
 
 	"k8s.io/apimachinery/pkg/types"
 )
 
 const (
 	losetupPath           = "losetup"
+	statPath              = "stat"
 	ErrDeviceNotFound     = "device not found"
 	ErrDeviceNotSupported = "device not supported"
 )
@@ -36,25 +39,28 @@ const (
 // BlockVolumePathHandler defines a set of operations for handling block volume-related operations
 type BlockVolumePathHandler interface {
 	// MapDevice creates a symbolic link to block device under specified map path
-	MapDevice(devicePath string, mapPath string, linkName string) error
+	MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error
 	// UnmapDevice removes a symbolic link to block device under specified map path
-	UnmapDevice(mapPath string, linkName string) error
+	UnmapDevice(mapPath string, linkName string, bindMount bool) error
 	// RemovePath removes a file or directory on specified map path
 	RemoveMapPath(mapPath string) error
 	// IsSymlinkExist retruns true if specified symbolic link exists
 	IsSymlinkExist(mapPath string) (bool, error)
-	// GetDeviceSymlinkRefs searches symbolic links under global map path
-	GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error)
+	// IsDeviceBindMountExist retruns true if specified bind mount exists
+	IsDeviceBindMountExist(mapPath string) (bool, error)
+	// GetDeviceBindMountRefs searches bind mounts under global map path
+	GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error)
 	// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
 	// corresponding to map path symlink, and then return global map path with pod uuid.
 	FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error)
 	// AttachFileDevice takes a path to a regular file and makes it available as an
 	// attached block device.
 	AttachFileDevice(path string) (string, error)
+	// DetachFileDevice takes a path to the attached block device and
+	// detach it from block device.
+	DetachFileDevice(path string) error
 	// GetLoopDevice returns the full path to the loop device associated with the given path.
 	GetLoopDevice(path string) (string, error)
-	// RemoveLoopDevice removes specified loopback device
-	RemoveLoopDevice(device string) error
 }
 
 // NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler.
@@ -68,7 +74,7 @@ type VolumePathHandler struct {
 }
 
 // MapDevice creates a symbolic link to block device under specified map path
-func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string) error {
+func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string, bindMount bool) error {
 	// Example of global map path:
 	//   globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid}
 	//   linkName: {podUid}
@@ -77,13 +83,13 @@ func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName
 	//   podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
 	//   linkName: {volumeName}
 	if len(devicePath) == 0 {
-		return fmt.Errorf("Failed to map device to map path. devicePath is empty")
+		return fmt.Errorf("failed to map device to map path. devicePath is empty")
 	}
 	if len(mapPath) == 0 {
-		return fmt.Errorf("Failed to map device to map path. mapPath is empty")
+		return fmt.Errorf("failed to map device to map path. mapPath is empty")
 	}
 	if !filepath.IsAbs(mapPath) {
-		return fmt.Errorf("The map path should be absolute: map path: %s", mapPath)
+		return fmt.Errorf("the map path should be absolute: map path: %s", mapPath)
 	}
 	klog.V(5).Infof("MapDevice: devicePath %s", devicePath)
 	klog.V(5).Infof("MapDevice: mapPath %s", mapPath)
@@ -92,31 +98,119 @@ func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName
 	// Check and create mapPath
 	_, err := os.Stat(mapPath)
 	if err != nil && !os.IsNotExist(err) {
-		klog.Errorf("cannot validate map path: %s", mapPath)
-		return err
+		return fmt.Errorf("cannot validate map path: %s: %v", mapPath, err)
 	}
 	if err = os.MkdirAll(mapPath, 0750); err != nil {
-		return fmt.Errorf("Failed to mkdir %s, error %v", mapPath, err)
+		return fmt.Errorf("failed to mkdir %s: %v", mapPath, err)
 	}
+
+	if bindMount {
+		return mapBindMountDevice(v, devicePath, mapPath, linkName)
+	}
+	return mapSymlinkDevice(v, devicePath, mapPath, linkName)
+}
+
+func mapBindMountDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
+	// Check bind mount exists
+	linkPath := filepath.Join(mapPath, string(linkName))
+
+	file, err := os.Stat(linkPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("failed to stat file %s: %v", linkPath, err)
+		}
+
+		// Create file
+		newFile, err := os.OpenFile(linkPath, os.O_CREATE|os.O_RDWR, 0750)
+		if err != nil {
+			return fmt.Errorf("failed to open file %s: %v", linkPath, err)
+		}
+		if err := newFile.Close(); err != nil {
+			return fmt.Errorf("failed to close file %s: %v", linkPath, err)
+		}
+	} else {
+		// Check if device file
+		// TODO: Need to check if this device file is actually the expected bind mount
+		if file.Mode()&os.ModeDevice == os.ModeDevice {
+			klog.Warningf("Warning: Map skipped because bind mount already exist on the path: %v", linkPath)
+			return nil
+		}
+
+		klog.Warningf("Warning: file %s is already exist but not mounted, skip creating file", linkPath)
+	}
+
+	// Bind mount file
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
+	if err := mounter.Mount(devicePath, linkPath, "" /* fsType */, []string{"bind"}); err != nil {
+		return fmt.Errorf("failed to bind mount devicePath: %s to linkPath %s: %v", devicePath, linkPath, err)
+	}
+
+	return nil
+}
+
+func mapSymlinkDevice(v VolumePathHandler, devicePath string, mapPath string, linkName string) error {
 	// Remove old symbolic link(or file) then create new one.
 	// This should be done because current symbolic link is
 	// stale across node reboot.
 	linkPath := filepath.Join(mapPath, string(linkName))
-	if err = os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
-		return err
+	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
 	}
-	err = os.Symlink(devicePath, linkPath)
-	return err
+	return os.Symlink(devicePath, linkPath)
 }
 
 // UnmapDevice removes a symbolic link associated to block device under specified map path
-func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error {
+func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string, bindMount bool) error {
 	if len(mapPath) == 0 {
-		return fmt.Errorf("Failed to unmap device from map path. mapPath is empty")
+		return fmt.Errorf("failed to unmap device from map path. mapPath is empty")
 	}
 	klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath)
 	klog.V(5).Infof("UnmapDevice: linkName %s", linkName)
 
+	if bindMount {
+		return unmapBindMountDevice(v, mapPath, linkName)
+	}
+	return unmapSymlinkDevice(v, mapPath, linkName)
+}
+
+func unmapBindMountDevice(v VolumePathHandler, mapPath string, linkName string) error {
+	// Check bind mount exists
+	linkPath := filepath.Join(mapPath, string(linkName))
+	if isMountExist, checkErr := v.IsDeviceBindMountExist(linkPath); checkErr != nil {
+		return checkErr
+	} else if !isMountExist {
+		klog.Warningf("Warning: Unmap skipped because bind mount does not exist on the path: %v", linkPath)
+
+		// Check if linkPath still exists
+		if _, err := os.Stat(linkPath); err != nil {
+			if !os.IsNotExist(err) {
+				return fmt.Errorf("failed to check if path %s exists: %v", linkPath, err)
+			}
+			// linkPath has already been removed
+			return nil
+		}
+		// Remove file
+		if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
+		}
+		return nil
+	}
+
+	// Unmount file
+	mounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: utilexec.New()}
+	if err := mounter.Unmount(linkPath); err != nil {
+		return fmt.Errorf("failed to unmount linkPath %s: %v", linkPath, err)
+	}
+
+	// Remove file
+	if err := os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove file %s: %v", linkPath, err)
+	}
+
+	return nil
+}
+
+func unmapSymlinkDevice(v VolumePathHandler, mapPath string, linkName string) error {
 	// Check symbolic link exists
 	linkPath := filepath.Join(mapPath, string(linkName))
 	if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil {
@@ -125,19 +219,18 @@ func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error {
 		klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath)
 		return nil
 	}
-	err := os.Remove(linkPath)
-	return err
+	return os.Remove(linkPath)
 }
 
 // RemoveMapPath removes a file or directory on specified map path
 func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
 	if len(mapPath) == 0 {
-		return fmt.Errorf("Failed to remove map path. mapPath is empty")
+		return fmt.Errorf("failed to remove map path. mapPath is empty")
 	}
 	klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath)
 	err := os.RemoveAll(mapPath)
 	if err != nil && !os.IsNotExist(err) {
-		return err
+		return fmt.Errorf("failed to remove directory %s: %v", mapPath, err)
 	}
 	return nil
 }
@@ -147,86 +240,59 @@ func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
 // On other cases, return false with error from Lstat().
 func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
 	fi, err := os.Lstat(mapPath)
-	if err == nil {
-		// If file exits and it's symbolic link, return true and no error
-		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
-			return true, nil
+	if err != nil {
+		// If file doesn't exist, return false and no error
+		if os.IsNotExist(err) {
+			return false, nil
 		}
-		// If file exits but it's not symbolic link, return fale and no error
-		return false, nil
+		// Return error from Lstat()
+		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
 	}
-	// If file doesn't exist, return false and no error
-	if os.IsNotExist(err) {
-		return false, nil
+	// If file exits and it's symbolic link, return true and no error
+	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+		return true, nil
 	}
-	// Return error from Lstat()
-	return false, err
+	// If file exits but it's not symbolic link, return false and no error
+	return false, nil
 }
 
-// GetDeviceSymlinkRefs searches symbolic links under global map path
-func (v VolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) {
+// IsDeviceBindMountExist returns true if specified file exists and the type is device.
+// If file doesn't exist, or file exists but not device, return false with no error.
+// On other cases, return false with error from Lstat().
+func (v VolumePathHandler) IsDeviceBindMountExist(mapPath string) (bool, error) {
+	fi, err := os.Lstat(mapPath)
+	if err != nil {
+		// If file doesn't exist, return false and no error
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+
+		// Return error from Lstat()
+		return false, fmt.Errorf("failed to Lstat file %s: %v", mapPath, err)
+	}
+	// If file exits and it's device, return true and no error
+	if fi.Mode()&os.ModeDevice == os.ModeDevice {
+		return true, nil
+	}
+	// If file exits but it's not device, return false and no error
+	return false, nil
+}
+
+// GetDeviceBindMountRefs searches bind mounts under global map path
+func (v VolumePathHandler) GetDeviceBindMountRefs(devPath string, mapPath string) ([]string, error) {
 	var refs []string
 	files, err := ioutil.ReadDir(mapPath)
 	if err != nil {
-		return nil, fmt.Errorf("Directory cannot read %v", err)
+		return nil, fmt.Errorf("directory cannot read %v", err)
 	}
 	for _, file := range files {
-		if file.Mode()&os.ModeSymlink != os.ModeSymlink {
+		if file.Mode()&os.ModeDevice != os.ModeDevice {
 			continue
 		}
 		filename := file.Name()
-		fp, err := os.Readlink(filepath.Join(mapPath, filename))
-		if err != nil {
-			return nil, fmt.Errorf("Symbolic link cannot be retrieved %v", err)
-		}
-		klog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", fp, devPath)
-		if fp == devPath {
-			refs = append(refs, filepath.Join(mapPath, filename))
-		}
+		// TODO: Might need to check if the file is actually linked to devPath
+		refs = append(refs, filepath.Join(mapPath, filename))
 	}
-	klog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs)
+	klog.V(5).Infof("GetDeviceBindMountRefs: refs %v", refs)
 	return refs, nil
 }
-
-// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
-// corresponding to map path symlink, and then return global map path with pod uuid.
-// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX
-//     globalMapPath/{pod uuid}: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX
-func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
-	var globalMapPathUUID string
-	// Find symbolic link named pod uuid under plugin dir
-	err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if (fi.Mode()&os.ModeSymlink == os.ModeSymlink) && (fi.Name() == string(podUID)) {
-			klog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath)
-			if res, err := compareSymlinks(path, mapPath); err == nil && res {
-				globalMapPathUUID = path
-			}
-		}
-		return nil
-	})
-	if err != nil {
-		return "", err
-	}
-	klog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID)
-	// Return path contains global map path + {pod uuid}
-	return globalMapPathUUID, nil
-}
-
-func compareSymlinks(global, pod string) (bool, error) {
-	devGlobal, err := os.Readlink(global)
-	if err != nil {
-		return false, err
-	}
-	devPod, err := os.Readlink(pod)
-	if err != nil {
-		return false, err
-	}
-	klog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod)
-	if devGlobal == devPod {
-		return true, nil
-	}
-	return false, nil
-}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go

@@ -19,12 +19,17 @@ limitations under the License.
 package volumepathhandler
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
+	"golang.org/x/sys/unix"
+
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
 )
 
@@ -33,7 +38,7 @@ import (
 func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
 	blockDevicePath, err := v.GetLoopDevice(path)
 	if err != nil && err.Error() != ErrDeviceNotFound {
-		return "", err
+		return "", fmt.Errorf("GetLoopDevice failed for path %s: %v", path, err)
 	}
 
 	// If no existing loop device for the path, create one
@@ -41,12 +46,33 @@ func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
 		klog.V(4).Infof("Creating device for path: %s", path)
 		blockDevicePath, err = makeLoopDevice(path)
 		if err != nil {
-			return "", err
+			return "", fmt.Errorf("makeLoopDevice failed for path %s: %v", path, err)
 		}
 	}
 	return blockDevicePath, nil
 }
 
+// DetachFileDevice takes a path to the attached block device and
+// detach it from block device.
+func (v VolumePathHandler) DetachFileDevice(path string) error {
+	loopPath, err := v.GetLoopDevice(path)
+	if err != nil {
+		if err.Error() == ErrDeviceNotFound {
+			klog.Warningf("couldn't find loopback device which takes file descriptor lock. Skip detaching device. device path: %q", path)
+		} else {
+			return fmt.Errorf("GetLoopDevice failed for path %s: %v", path, err)
+		}
+	} else {
+		if len(loopPath) != 0 {
+			err = removeLoopDevice(loopPath)
+			if err != nil {
+				return fmt.Errorf("removeLoopDevice failed for path %s: %v", path, err)
+			}
+		}
+	}
+	return nil
+}
+
 // GetLoopDevice returns the full path to the loop device associated with the given path.
 func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
 	_, err := os.Stat(path)
@@ -62,9 +88,9 @@ func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		klog.V(2).Infof("Failed device discover command for path %s: %v %s", path, err, out)
-		return "", err
+		return "", fmt.Errorf("losetup -j %s failed: %v", path, err)
 	}
-	return parseLosetupOutputForDevice(out)
+	return parseLosetupOutputForDevice(out, path)
 }
 
 func makeLoopDevice(path string) (string, error) {
@@ -73,13 +99,20 @@ func makeLoopDevice(path string) (string, error) {
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		klog.V(2).Infof("Failed device create command for path: %s %v %s ", path, err, out)
-		return "", err
+		return "", fmt.Errorf("losetup -f --show %s failed: %v", path, err)
 	}
-	return parseLosetupOutputForDevice(out)
+
+	// losetup -f --show {path} returns device in the format:
+	// /dev/loop1
+	if len(out) == 0 {
+		return "", errors.New(ErrDeviceNotFound)
+	}
+
+	return strings.TrimSpace(string(out)), nil
 }
 
-// RemoveLoopDevice removes specified loopback device
-func (v VolumePathHandler) RemoveLoopDevice(device string) error {
+// removeLoopDevice removes specified loopback device
+func removeLoopDevice(device string) error {
 	args := []string{"-d", device}
 	cmd := exec.Command(losetupPath, args...)
 	out, err := cmd.CombinedOutput()
@@ -88,21 +121,121 @@ func (v VolumePathHandler) RemoveLoopDevice(device string) error {
 			return nil
 		}
 		klog.V(2).Infof("Failed to remove loopback device: %s: %v %s", device, err, out)
-		return err
+		return fmt.Errorf("losetup -d %s failed: %v", device, err)
 	}
 	return nil
 }
 
-func parseLosetupOutputForDevice(output []byte) (string, error) {
+func parseLosetupOutputForDevice(output []byte, path string) (string, error) {
 	if len(output) == 0 {
 		return "", errors.New(ErrDeviceNotFound)
 	}
 
-	// losetup returns device in the format:
-	// /dev/loop1: [0073]:148662 (/dev/sda)
-	device := strings.TrimSpace(strings.SplitN(string(output), ":", 2)[0])
+	// losetup -j {path} returns device in the format:
+	// /dev/loop1: [0073]:148662 ({path})
+	// /dev/loop2: [0073]:148662 (/dev/sdX)
+	//
+	// losetup -j shows all the loop device for the same device that has the same
+	// major/minor number, by resolving symlink and matching major/minor number.
+	// Therefore, there will be other path than {path} in output, as shown in above output.
+	s := string(output)
+	// Find the line that exact matches to the path, or "({path})"
+	var matched string
+	scanner := bufio.NewScanner(strings.NewReader(s))
+	for scanner.Scan() {
+		if strings.HasSuffix(scanner.Text(), "("+path+")") {
+			matched = scanner.Text()
+			break
+		}
+	}
+	if len(matched) == 0 {
+		return "", errors.New(ErrDeviceNotFound)
+	}
+	s = matched
+
+	// Get device name, or the 0th field of the output separated with ":".
+	// We don't need 1st field or later to be splitted, so passing 2 to SplitN.
+	device := strings.TrimSpace(strings.SplitN(s, ":", 2)[0])
 	if len(device) == 0 {
 		return "", errors.New(ErrDeviceNotFound)
 	}
 	return device, nil
 }
+
+// FindGlobalMapPathUUIDFromPod finds {pod uuid} bind mount under globalMapPath
+// corresponding to map path symlink, and then return global map path with pod uuid.
+// (See pkg/volume/volume.go for details on a global map path and a pod device map path.)
+// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX
+//     globalMapPath/{pod uuid} bind mount: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX
+func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
+	var globalMapPathUUID string
+	// Find symbolic link named pod uuid under plugin dir
+	err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if (fi.Mode()&os.ModeDevice == os.ModeDevice) && (fi.Name() == string(podUID)) {
+			klog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath)
+			if res, err := compareBindMountAndSymlinks(path, mapPath); err == nil && res {
+				globalMapPathUUID = path
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", fmt.Errorf("FindGlobalMapPathUUIDFromPod failed: %v", err)
+	}
+	klog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID)
+	// Return path contains global map path + {pod uuid}
+	return globalMapPathUUID, nil
+}
+
+// compareBindMountAndSymlinks returns if global path (bind mount) and
+// pod path (symlink) are pointing to the same device.
+// If there is an error in checking it returns error.
+func compareBindMountAndSymlinks(global, pod string) (bool, error) {
+	// To check if bind mount and symlink are pointing to the same device,
+	// we need to check if they are pointing to the devices that have same major/minor number.
+
+	// Get the major/minor number for global path
+	devNumGlobal, err := getDeviceMajorMinor(global)
+	if err != nil {
+		return false, fmt.Errorf("getDeviceMajorMinor failed for path %s: %v", global, err)
+	}
+
+	// Get the symlinked device from the pod path
+	devPod, err := os.Readlink(pod)
+	if err != nil {
+		return false, fmt.Errorf("failed to readlink path %s: %v", pod, err)
+	}
+	// Get the major/minor number for the symlinked device from the pod path
+	devNumPod, err := getDeviceMajorMinor(devPod)
+	if err != nil {
+		return false, fmt.Errorf("getDeviceMajorMinor failed for path %s: %v", devPod, err)
+	}
+	klog.V(5).Infof("CompareBindMountAndSymlinks: devNumGlobal %s, devNumPod %s", devNumGlobal, devNumPod)
+
+	// Check if the major/minor number are the same
+	if devNumGlobal == devNumPod {
+		return true, nil
+	}
+	return false, nil
+}
+
+// getDeviceMajorMinor returns major/minor number for the path with below format:
+// major:minor (in hex)
+// ex)
+//     fc:10
+func getDeviceMajorMinor(path string) (string, error) {
+	var stat unix.Stat_t
+
+	if err := unix.Stat(path, &stat); err != nil {
+		return "", fmt.Errorf("failed to stat path %s: %v", path, err)
+	}
+
+	devNumber := uint64(stat.Rdev)
+	major := unix.Major(devNumber)
+	minor := unix.Minor(devNumber)
+
+	return fmt.Sprintf("%x:%x", major, minor), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler_unsupported.go

@@ -20,6 +20,8 @@ package volumepathhandler
 
 import (
 	"fmt"
+
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // AttachFileDevice takes a path to a regular file and makes it available as an
@@ -28,12 +30,19 @@ func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
 	return "", fmt.Errorf("AttachFileDevice not supported for this build.")
 }
 
+// DetachFileDevice takes a path to the attached block device and
+// detach it from block device.
+func (v VolumePathHandler) DetachFileDevice(path string) error {
+	return fmt.Errorf("DetachFileDevice not supported for this build.")
+}
+
 // GetLoopDevice returns the full path to the loop device associated with the given path.
 func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
 	return "", fmt.Errorf("GetLoopDevice not supported for this build.")
 }
 
-// RemoveLoopDevice removes specified loopback device
-func (v VolumePathHandler) RemoveLoopDevice(device string) error {
-	return fmt.Errorf("RemoveLoopDevice not supported for this build.")
+// FindGlobalMapPathUUIDFromPod finds {pod uuid} bind mount under globalMapPath
+// corresponding to map path symlink, and then return global map path with pod uuid.
+func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
+	return "", fmt.Errorf("FindGlobalMapPathUUIDFromPod not supported for this build.")
 }