vendor update for E2E framework

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

vendor/k8s.io/kubernetes/pkg/volume/util/atomic_writer.go

@@ -0,0 +1,454 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"k8s.io/klog"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+const (
+	maxFileNameLength = 255
+	maxPathLength     = 4096
+)
+
+// AtomicWriter handles atomically projecting content for a set of files into
+// a target directory.
+//
+// Note:
+//
+// 1. AtomicWriter reserves the set of pathnames starting with `..`.
+// 2. AtomicWriter offers no concurrency guarantees and must be synchronized
+//    by the caller.
+//
+// The visible files in this volume are symlinks to files in the writer's data
+// directory.  Actual files are stored in a hidden timestamped directory which
+// is symlinked to by the data directory. The timestamped directory and
+// data directory symlink are created in the writer's target dir.  This scheme
+// allows the files to be atomically updated by changing the target of the
+// data directory symlink.
+//
+// Consumers of the target directory can monitor the ..data symlink using
+// inotify or fanotify to receive events when the content in the volume is
+// updated.
+type AtomicWriter struct {
+	targetDir  string
+	logContext string
+}
+
+// FileProjection contains file Data and access Mode
+type FileProjection struct {
+	Data []byte
+	Mode int32
+}
+
+// NewAtomicWriter creates a new AtomicWriter configured to write to the given
+// target directory, or returns an error if the target directory does not exist.
+func NewAtomicWriter(targetDir string, logContext string) (*AtomicWriter, error) {
+	_, err := os.Stat(targetDir)
+	if os.IsNotExist(err) {
+		return nil, err
+	}
+
+	return &AtomicWriter{targetDir: targetDir, logContext: logContext}, nil
+}
+
+const (
+	dataDirName    = "..data"
+	newDataDirName = "..data_tmp"
+)
+
+// Write does an atomic projection of the given payload into the writer's target
+// directory.  Input paths must not begin with '..'.
+//
+// The Write algorithm is:
+//
+//  1.  The payload is validated; if the payload is invalid, the function returns
+//  2.  The current timestamped directory is detected by reading the data directory
+//      symlink
+//  3.  The old version of the volume is walked to determine whether any
+//      portion of the payload was deleted and is still present on disk.
+//  4.  The data in the current timestamped directory is compared to the projected
+//      data to determine if an update is required.
+//  5.  A new timestamped dir is created
+//  6.  The payload is written to the new timestamped directory
+//  7.  Symlinks and directory for new user-visible files are created (if needed).
+//
+//      For example, consider the files:
+//        <target-dir>/podName
+//        <target-dir>/user/labels
+//        <target-dir>/k8s/annotations
+//
+//      The user visible files are symbolic links into the internal data directory:
+//        <target-dir>/podName         -> ..data/podName
+//        <target-dir>/usr -> ..data/usr
+//        <target-dir>/k8s -> ..data/k8s
+//
+//      The data directory itself is a link to a timestamped directory with
+//      the real data:
+//        <target-dir>/..data          -> ..2016_02_01_15_04_05.12345678/
+//  8.  A symlink to the new timestamped directory ..data_tmp is created that will
+//      become the new data directory
+//  9.  The new data directory symlink is renamed to the data directory; rename is atomic
+// 10.  Old paths are removed from the user-visible portion of the target directory
+// 11.  The previous timestamped directory is removed, if it exists
+func (w *AtomicWriter) Write(payload map[string]FileProjection) error {
+	// (1)
+	cleanPayload, err := validatePayload(payload)
+	if err != nil {
+		klog.Errorf("%s: invalid payload: %v", w.logContext, err)
+		return err
+	}
+
+	// (2)
+	dataDirPath := path.Join(w.targetDir, dataDirName)
+	oldTsDir, err := os.Readlink(dataDirPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			klog.Errorf("%s: error reading link for data directory: %v", w.logContext, err)
+			return err
+		}
+		// although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs)
+		// empty oldTsDir indicates that it didn't exist
+		oldTsDir = ""
+	}
+	oldTsPath := path.Join(w.targetDir, oldTsDir)
+
+	var pathsToRemove sets.String
+	// if there was no old version, there's nothing to remove
+	if len(oldTsDir) != 0 {
+		// (3)
+		pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath)
+		if err != nil {
+			klog.Errorf("%s: error determining user-visible files to remove: %v", w.logContext, err)
+			return err
+		}
+
+		// (4)
+		if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil {
+			klog.Errorf("%s: error determining whether payload should be written to disk: %v", w.logContext, err)
+			return err
+		} else if !should && len(pathsToRemove) == 0 {
+			klog.V(4).Infof("%s: no update required for target directory %v", w.logContext, w.targetDir)
+			return nil
+		} else {
+			klog.V(4).Infof("%s: write required for target directory %v", w.logContext, w.targetDir)
+		}
+	}
+
+	// (5)
+	tsDir, err := w.newTimestampDir()
+	if err != nil {
+		klog.V(4).Infof("%s: error creating new ts data directory: %v", w.logContext, err)
+		return err
+	}
+	tsDirName := filepath.Base(tsDir)
+
+	// (6)
+	if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil {
+		klog.Errorf("%s: error writing payload to ts data directory %s: %v", w.logContext, tsDir, err)
+		return err
+	}
+	klog.V(4).Infof("%s: performed write of new data to ts data directory: %s", w.logContext, tsDir)
+
+	// (7)
+	if err = w.createUserVisibleFiles(cleanPayload); err != nil {
+		klog.Errorf("%s: error creating visible symlinks in %s: %v", w.logContext, w.targetDir, err)
+		return err
+	}
+
+	// (8)
+	newDataDirPath := path.Join(w.targetDir, newDataDirName)
+	if err = os.Symlink(tsDirName, newDataDirPath); err != nil {
+		os.RemoveAll(tsDir)
+		klog.Errorf("%s: error creating symbolic link for atomic update: %v", w.logContext, err)
+		return err
+	}
+
+	// (9)
+	if runtime.GOOS == "windows" {
+		os.Remove(dataDirPath)
+		err = os.Symlink(tsDirName, dataDirPath)
+		os.Remove(newDataDirPath)
+	} else {
+		err = os.Rename(newDataDirPath, dataDirPath)
+	}
+	if err != nil {
+		os.Remove(newDataDirPath)
+		os.RemoveAll(tsDir)
+		klog.Errorf("%s: error renaming symbolic link for data directory %s: %v", w.logContext, newDataDirPath, err)
+		return err
+	}
+
+	// (10)
+	if err = w.removeUserVisiblePaths(pathsToRemove); err != nil {
+		klog.Errorf("%s: error removing old visible symlinks: %v", w.logContext, err)
+		return err
+	}
+
+	// (11)
+	if len(oldTsDir) > 0 {
+		if err = os.RemoveAll(oldTsPath); err != nil {
+			klog.Errorf("%s: error removing old data directory %s: %v", w.logContext, oldTsDir, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// validatePayload returns an error if any path in the payload returns a copy of the payload with the paths cleaned.
+func validatePayload(payload map[string]FileProjection) (map[string]FileProjection, error) {
+	cleanPayload := make(map[string]FileProjection)
+	for k, content := range payload {
+		if err := validatePath(k); err != nil {
+			return nil, err
+		}
+
+		cleanPayload[filepath.Clean(k)] = content
+	}
+
+	return cleanPayload, nil
+}
+
+// validatePath validates a single path, returning an error if the path is
+// invalid.  paths may not:
+//
+// 1. be absolute
+// 2. contain '..' as an element
+// 3. start with '..'
+// 4. contain filenames larger than 255 characters
+// 5. be longer than 4096 characters
+func validatePath(targetPath string) error {
+	// TODO: somehow unify this with the similar api validation,
+	// validateVolumeSourcePath; the error semantics are just different enough
+	// from this that it was time-prohibitive trying to find the right
+	// refactoring to re-use.
+	if targetPath == "" {
+		return fmt.Errorf("invalid path: must not be empty: %q", targetPath)
+	}
+	if path.IsAbs(targetPath) {
+		return fmt.Errorf("invalid path: must be relative path: %s", targetPath)
+	}
+
+	if len(targetPath) > maxPathLength {
+		return fmt.Errorf("invalid path: must be less than or equal to %d characters", maxPathLength)
+	}
+
+	items := strings.Split(targetPath, string(os.PathSeparator))
+	for _, item := range items {
+		if item == ".." {
+			return fmt.Errorf("invalid path: must not contain '..': %s", targetPath)
+		}
+		if len(item) > maxFileNameLength {
+			return fmt.Errorf("invalid path: filenames must be less than or equal to %d characters", maxFileNameLength)
+		}
+	}
+	if strings.HasPrefix(items[0], "..") && len(items[0]) > 2 {
+		return fmt.Errorf("invalid path: must not start with '..': %s", targetPath)
+	}
+
+	return nil
+}
+
+// shouldWritePayload returns whether the payload should be written to disk.
+func shouldWritePayload(payload map[string]FileProjection, oldTsDir string) (bool, error) {
+	for userVisiblePath, fileProjection := range payload {
+		shouldWrite, err := shouldWriteFile(path.Join(oldTsDir, userVisiblePath), fileProjection.Data)
+		if err != nil {
+			return false, err
+		}
+
+		if shouldWrite {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// shouldWriteFile returns whether a new version of a file should be written to disk.
+func shouldWriteFile(path string, content []byte) (bool, error) {
+	_, err := os.Lstat(path)
+	if os.IsNotExist(err) {
+		return true, nil
+	}
+
+	contentOnFs, err := ioutil.ReadFile(path)
+	if err != nil {
+		return false, err
+	}
+
+	return (bytes.Compare(content, contentOnFs) != 0), nil
+}
+
+// pathsToRemove walks the current version of the data directory and
+// determines which paths should be removed (if any) after the payload is
+// written to the target directory.
+func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) {
+	paths := sets.NewString()
+	visitor := func(path string, info os.FileInfo, err error) error {
+		relativePath := strings.TrimPrefix(path, oldTsDir)
+		relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator))
+		if relativePath == "" {
+			return nil
+		}
+
+		paths.Insert(relativePath)
+		return nil
+	}
+
+	err := filepath.Walk(oldTsDir, visitor)
+	if os.IsNotExist(err) {
+		return nil, nil
+	} else if err != nil {
+		return nil, err
+	}
+	klog.V(5).Infof("%s: current paths:   %+v", w.targetDir, paths.List())
+
+	newPaths := sets.NewString()
+	for file := range payload {
+		// add all subpaths for the payload to the set of new paths
+		// to avoid attempting to remove non-empty dirs
+		for subPath := file; subPath != ""; {
+			newPaths.Insert(subPath)
+			subPath, _ = filepath.Split(subPath)
+			subPath = strings.TrimSuffix(subPath, string(os.PathSeparator))
+		}
+	}
+	klog.V(5).Infof("%s: new paths:       %+v", w.targetDir, newPaths.List())
+
+	result := paths.Difference(newPaths)
+	klog.V(5).Infof("%s: paths to remove: %+v", w.targetDir, result)
+
+	return result, nil
+}
+
+// newTimestampDir creates a new timestamp directory
+func (w *AtomicWriter) newTimestampDir() (string, error) {
+	tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05."))
+	if err != nil {
+		klog.Errorf("%s: unable to create new temp directory: %v", w.logContext, err)
+		return "", err
+	}
+
+	// 0755 permissions are needed to allow 'group' and 'other' to recurse the
+	// directory tree.  do a chmod here to ensure that permissions are set correctly
+	// regardless of the process' umask.
+	err = os.Chmod(tsDir, 0755)
+	if err != nil {
+		klog.Errorf("%s: unable to set mode on new temp directory: %v", w.logContext, err)
+		return "", err
+	}
+
+	return tsDir, nil
+}
+
+// writePayloadToDir writes the given payload to the given directory.  The
+// directory must exist.
+func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir string) error {
+	for userVisiblePath, fileProjection := range payload {
+		content := fileProjection.Data
+		mode := os.FileMode(fileProjection.Mode)
+		fullPath := path.Join(dir, userVisiblePath)
+		baseDir, _ := filepath.Split(fullPath)
+
+		err := os.MkdirAll(baseDir, os.ModePerm)
+		if err != nil {
+			klog.Errorf("%s: unable to create directory %s: %v", w.logContext, baseDir, err)
+			return err
+		}
+
+		err = ioutil.WriteFile(fullPath, content, mode)
+		if err != nil {
+			klog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err)
+			return err
+		}
+		// Chmod is needed because ioutil.WriteFile() ends up calling
+		// open(2) to create the file, so the final mode used is "mode &
+		// ~umask". But we want to make sure the specified mode is used
+		// in the file no matter what the umask is.
+		err = os.Chmod(fullPath, mode)
+		if err != nil {
+			klog.Errorf("%s: unable to write file %s with mode %v: %v", w.logContext, fullPath, mode, err)
+		}
+	}
+
+	return nil
+}
+
+// createUserVisibleFiles creates the relative symlinks for all the
+// files configured in the payload. If the directory in a file path does not
+// exist, it is created.
+//
+// Viz:
+// For files: "bar", "foo/bar", "baz/bar", "foo/baz/blah"
+// the following symlinks are created:
+// bar -> ..data/bar
+// foo -> ..data/foo
+// baz -> ..data/baz
+func (w *AtomicWriter) createUserVisibleFiles(payload map[string]FileProjection) error {
+	for userVisiblePath := range payload {
+		slashpos := strings.Index(userVisiblePath, string(os.PathSeparator))
+		if slashpos == -1 {
+			slashpos = len(userVisiblePath)
+		}
+		linkname := userVisiblePath[:slashpos]
+		_, err := os.Readlink(path.Join(w.targetDir, linkname))
+		if err != nil && os.IsNotExist(err) {
+			// The link into the data directory for this path doesn't exist; create it
+			visibleFile := path.Join(w.targetDir, linkname)
+			dataDirFile := path.Join(dataDirName, linkname)
+
+			err = os.Symlink(dataDirFile, visibleFile)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// removeUserVisiblePaths removes the set of paths from the user-visible
+// portion of the writer's target directory.
+func (w *AtomicWriter) removeUserVisiblePaths(paths sets.String) error {
+	ps := string(os.PathSeparator)
+	var lasterr error
+	for p := range paths {
+		// only remove symlinks from the volume root directory (i.e. items that don't contain '/')
+		if strings.Contains(p, ps) {
+			continue
+		}
+		if err := os.Remove(path.Join(w.targetDir, p)); err != nil {
+			klog.Errorf("%s: error pruning old user-visible path %s: %v", w.logContext, p, err)
+			lasterr = err
+		}
+	}
+
+	return lasterr
+}

vendor/k8s.io/kubernetes/pkg/volume/util/attach_limit.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+)
+
+// This file is a common place holder for volume limit utility constants
+// shared between volume package and scheduler
+
+const (
+	// EBSVolumeLimitKey resource name that will store volume limits for EBS
+	EBSVolumeLimitKey = "attachable-volumes-aws-ebs"
+	// EBSNitroLimitRegex finds nitro instance types with different limit than EBS defaults
+	EBSNitroLimitRegex = "^[cmr]5.*|t3|z1d"
+	// DefaultMaxEBSVolumes is the limit for volumes attached to an instance.
+	// Amazon recommends no more than 40; the system root volume uses at least one.
+	// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/volume_limits.html#linux-specific-volume-limits
+	DefaultMaxEBSVolumes = 39
+	// DefaultMaxEBSNitroVolumeLimit is default EBS volume limit on m5 and c5 instances
+	DefaultMaxEBSNitroVolumeLimit = 25
+	// AzureVolumeLimitKey stores resource name that will store volume limits for Azure
+	AzureVolumeLimitKey = "attachable-volumes-azure-disk"
+	// GCEVolumeLimitKey stores resource name that will store volume limits for GCE node
+	GCEVolumeLimitKey = "attachable-volumes-gce-pd"
+
+	// CinderVolumeLimitKey contains Volume limit key for Cinder
+	CinderVolumeLimitKey = "attachable-volumes-cinder"
+	// DefaultMaxCinderVolumes defines the maximum number of PD Volumes for Cinder
+	// For Openstack we are keeping this to a high enough value so as depending on backend
+	// cluster admins can configure it.
+	DefaultMaxCinderVolumes = 256
+
+	// CSIAttachLimitPrefix defines prefix used for CSI volumes
+	CSIAttachLimitPrefix = "attachable-volumes-csi-"
+
+	// ResourceNameLengthLimit stores maximum allowed Length for a ResourceName
+	ResourceNameLengthLimit = 63
+)
+
+// GetCSIAttachLimitKey returns limit key used for CSI volumes
+func GetCSIAttachLimitKey(driverName string) string {
+	csiPrefixLength := len(CSIAttachLimitPrefix)
+	totalkeyLength := csiPrefixLength + len(driverName)
+	if totalkeyLength >= ResourceNameLengthLimit {
+		charsFromDriverName := driverName[:23]
+		hash := sha1.New()
+		hash.Write([]byte(driverName))
+		hashed := hex.EncodeToString(hash.Sum(nil))
+		hashed = hashed[:16]
+		return CSIAttachLimitPrefix + charsFromDriverName + hashed
+	}
+	return CSIAttachLimitPrefix + driverName
+}

vendor/k8s.io/kubernetes/pkg/volume/util/device_util.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+//DeviceUtil is a util for common device methods
+type DeviceUtil interface {
+	FindMultipathDeviceForDevice(disk string) string
+	FindSlaveDevicesOnMultipath(disk string) []string
+	GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error)
+	FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error)
+}
+
+type deviceHandler struct {
+	getIo IoUtil
+}
+
+//NewDeviceHandler Create a new IoHandler implementation
+func NewDeviceHandler(io IoUtil) DeviceUtil {
+	return &deviceHandler{getIo: io}
+}

vendor/k8s.io/kubernetes/pkg/volume/util/device_util_linux.go

@@ -0,0 +1,299 @@
+// +build linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"k8s.io/klog"
+)
+
+// FindMultipathDeviceForDevice given a device name like /dev/sdx, find the devicemapper parent
+func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string {
+	io := handler.getIo
+	disk, err := findDeviceForPath(device, io)
+	if err != nil {
+		return ""
+	}
+	sysPath := "/sys/block/"
+	if dirs, err := io.ReadDir(sysPath); err == nil {
+		for _, f := range dirs {
+			name := f.Name()
+			if strings.HasPrefix(name, "dm-") {
+				if _, err1 := io.Lstat(sysPath + name + "/slaves/" + disk); err1 == nil {
+					return "/dev/" + name
+				}
+			}
+		}
+	}
+	return ""
+}
+
+// findDeviceForPath Find the underlaying disk for a linked path such as /dev/disk/by-path/XXXX or /dev/mapper/XXXX
+// will return sdX or hdX etc, if /dev/sdX is passed in then sdX will be returned
+func findDeviceForPath(path string, io IoUtil) (string, error) {
+	devicePath, err := io.EvalSymlinks(path)
+	if err != nil {
+		return "", err
+	}
+	// if path /dev/hdX split into "", "dev", "hdX" then we will
+	// return just the last part
+	parts := strings.Split(devicePath, "/")
+	if len(parts) == 3 && strings.HasPrefix(parts[1], "dev") {
+		return parts[2], nil
+	}
+	return "", errors.New("Illegal path for device " + devicePath)
+}
+
+// FindSlaveDevicesOnMultipath given a dm name like /dev/dm-1, find all devices
+// which are managed by the devicemapper dm-1.
+func (handler *deviceHandler) FindSlaveDevicesOnMultipath(dm string) []string {
+	var devices []string
+	io := handler.getIo
+	// Split path /dev/dm-1 into "", "dev", "dm-1"
+	parts := strings.Split(dm, "/")
+	if len(parts) != 3 || !strings.HasPrefix(parts[1], "dev") {
+		return devices
+	}
+	disk := parts[2]
+	slavesPath := path.Join("/sys/block/", disk, "/slaves/")
+	if files, err := io.ReadDir(slavesPath); err == nil {
+		for _, f := range files {
+			devices = append(devices, path.Join("/dev/", f.Name()))
+		}
+	}
+	return devices
+}
+
+// GetISCSIPortalHostMapForTarget given a target iqn, find all the scsi hosts logged into
+// that target. Returns a map of iSCSI portals (string) to SCSI host numbers (integers).
+// For example: {
+//    "192.168.30.7:3260": 2,
+//    "192.168.30.8:3260": 3,
+// }
+func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
+	portalHostMap := make(map[string]int)
+	io := handler.getIo
+
+	// Iterate over all the iSCSI hosts in sysfs
+	sysPath := "/sys/class/iscsi_host"
+	hostDirs, err := io.ReadDir(sysPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return portalHostMap, nil
+		}
+		return nil, err
+	}
+	for _, hostDir := range hostDirs {
+		// iSCSI hosts are always of the format "host%d"
+		// See drivers/scsi/hosts.c in Linux
+		hostName := hostDir.Name()
+		if !strings.HasPrefix(hostName, "host") {
+			continue
+		}
+		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
+		if err != nil {
+			klog.Errorf("Could not get number from iSCSI host: %s", hostName)
+			continue
+		}
+
+		// Iterate over the children of the iscsi_host device
+		// We are looking for the associated session
+		devicePath := sysPath + "/" + hostName + "/device"
+		deviceDirs, err := io.ReadDir(devicePath)
+		if err != nil {
+			return nil, err
+		}
+		for _, deviceDir := range deviceDirs {
+			// Skip over files that aren't the session
+			// Sessions are of the format "session%u"
+			// See drivers/scsi/scsi_transport_iscsi.c in Linux
+			sessionName := deviceDir.Name()
+			if !strings.HasPrefix(sessionName, "session") {
+				continue
+			}
+
+			sessionPath := devicePath + "/" + sessionName
+
+			// Read the target name for the iSCSI session
+			targetNamePath := sessionPath + "/iscsi_session/" + sessionName + "/targetname"
+			targetName, err := io.ReadFile(targetNamePath)
+			if err != nil {
+				klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
+				continue
+			}
+
+			// Ignore hosts that don't matchthe target we were looking for.
+			if strings.TrimSpace(string(targetName)) != targetIqn {
+				continue
+			}
+
+			// Iterate over the children of the iSCSI session looking
+			// for the iSCSI connection.
+			dirs2, err := io.ReadDir(sessionPath)
+			if err != nil {
+				klog.Infof("Failed to process session %s, assuming this session is unavailable: %s", sessionName, err)
+				continue
+			}
+			for _, dir2 := range dirs2 {
+				// Skip over files that aren't the connection
+				// Connections are of the format "connection%d:%u"
+				// See drivers/scsi/scsi_transport_iscsi.c in Linux
+				dirName := dir2.Name()
+				if !strings.HasPrefix(dirName, "connection") {
+					continue
+				}
+
+				connectionPath := sessionPath + "/" + dirName + "/iscsi_connection/" + dirName
+
+				// Read the current and persistent portal information for the connection.
+				addrPath := connectionPath + "/address"
+				addr, err := io.ReadFile(addrPath)
+				if err != nil {
+					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
+					continue
+				}
+
+				portPath := connectionPath + "/port"
+				port, err := io.ReadFile(portPath)
+				if err != nil {
+					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
+					continue
+				}
+
+				persistentAddrPath := connectionPath + "/persistent_address"
+				persistentAddr, err := io.ReadFile(persistentAddrPath)
+				if err != nil {
+					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
+					continue
+				}
+
+				persistentPortPath := connectionPath + "/persistent_port"
+				persistentPort, err := io.ReadFile(persistentPortPath)
+				if err != nil {
+					klog.Infof("Failed to process connection %s, assuming this connection is unavailable: %s", dirName, err)
+					continue
+				}
+
+				// Add entries to the map for both the current and persistent portals
+				// pointing to the SCSI host for those connections
+				portal := strings.TrimSpace(string(addr)) + ":" +
+					strings.TrimSpace(string(port))
+				portalHostMap[portal] = hostNumber
+
+				persistentPortal := strings.TrimSpace(string(persistentAddr)) + ":" +
+					strings.TrimSpace(string(persistentPort))
+				portalHostMap[persistentPortal] = hostNumber
+			}
+		}
+	}
+
+	return portalHostMap, nil
+}
+
+// FindDevicesForISCSILun given an iqn, and lun number, find all the devices
+// corresponding to that LUN.
+func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
+	devices := make([]string, 0)
+	io := handler.getIo
+
+	// Iterate over all the iSCSI hosts in sysfs
+	sysPath := "/sys/class/iscsi_host"
+	hostDirs, err := io.ReadDir(sysPath)
+	if err != nil {
+		return nil, err
+	}
+	for _, hostDir := range hostDirs {
+		// iSCSI hosts are always of the format "host%d"
+		// See drivers/scsi/hosts.c in Linux
+		hostName := hostDir.Name()
+		if !strings.HasPrefix(hostName, "host") {
+			continue
+		}
+		hostNumber, err := strconv.Atoi(strings.TrimPrefix(hostName, "host"))
+		if err != nil {
+			klog.Errorf("Could not get number from iSCSI host: %s", hostName)
+			continue
+		}
+
+		// Iterate over the children of the iscsi_host device
+		// We are looking for the associated session
+		devicePath := sysPath + "/" + hostName + "/device"
+		deviceDirs, err := io.ReadDir(devicePath)
+		if err != nil {
+			return nil, err
+		}
+		for _, deviceDir := range deviceDirs {
+			// Skip over files that aren't the session
+			// Sessions are of the format "session%u"
+			// See drivers/scsi/scsi_transport_iscsi.c in Linux
+			sessionName := deviceDir.Name()
+			if !strings.HasPrefix(sessionName, "session") {
+				continue
+			}
+
+			// Read the target name for the iSCSI session
+			targetNamePath := devicePath + "/" + sessionName + "/iscsi_session/" + sessionName + "/targetname"
+			targetName, err := io.ReadFile(targetNamePath)
+			if err != nil {
+				return nil, err
+			}
+
+			// Only if the session matches the target we were looking for,
+			// add it to the map
+			if strings.TrimSpace(string(targetName)) != targetIqn {
+				continue
+			}
+
+			// The list of block devices on the scsi bus will be in a
+			// directory called "target%d:%d:%d".
+			// See drivers/scsi/scsi_scan.c in Linux
+			// We assume the channel/bus and device/controller are always zero for iSCSI
+			targetPath := devicePath + "/" + sessionName + fmt.Sprintf("/target%d:0:0", hostNumber)
+
+			// The block device for a given lun will be "%d:%d:%d:%d" --
+			// host:channel:bus:LUN
+			blockDevicePath := targetPath + fmt.Sprintf("/%d:0:0:%d", hostNumber, lun)
+
+			// If the LUN doesn't exist on this bus, continue on
+			_, err = io.Lstat(blockDevicePath)
+			if err != nil {
+				continue
+			}
+
+			// Read the block directory, there should only be one child --
+			// the block device "sd*"
+			path := blockDevicePath + "/block"
+			dirs, err := io.ReadDir(path)
+			if err != nil {
+				return nil, err
+			}
+			if 0 < len(dirs) {
+				devices = append(devices, dirs[0].Name())
+			}
+		}
+	}
+
+	return devices, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/device_util_unsupported.go

@@ -0,0 +1,42 @@
+// +build !linux
+
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+// FindMultipathDeviceForDevice unsupported returns ""
+func (handler *deviceHandler) FindMultipathDeviceForDevice(device string) string {
+	return ""
+}
+
+// FindSlaveDevicesOnMultipath unsupported returns ""
+func (handler *deviceHandler) FindSlaveDevicesOnMultipath(disk string) []string {
+	out := []string{}
+	return out
+}
+
+// GetISCSIPortalHostMapForTarget unsupported returns nil
+func (handler *deviceHandler) GetISCSIPortalHostMapForTarget(targetIqn string) (map[string]int, error) {
+	portalHostMap := make(map[string]int)
+	return portalHostMap, nil
+}
+
+// FindDevicesForISCSILun unsupported returns nil
+func (handler *deviceHandler) FindDevicesForISCSILun(targetIqn string, lun int) ([]string, error) {
+	devices := []string{}
+	return devices, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package util contains utility code for use by volume plugins.
+package util // import "k8s.io/kubernetes/pkg/volume/util"

vendor/k8s.io/kubernetes/pkg/volume/util/finalizer.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+const (
+	// PVCProtectionFinalizer is the name of finalizer on PVCs that have a running pod.
+	PVCProtectionFinalizer = "kubernetes.io/pvc-protection"
+
+	// PVProtectionFinalizer is the name of finalizer on PVs that are bound by PVCs
+	PVProtectionFinalizer = "kubernetes.io/pv-protection"
+)

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs.go

@@ -0,0 +1,98 @@
+// +build linux darwin
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// FSInfo linux returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	statfs := &unix.Statfs_t{}
+	err := unix.Statfs(path, statfs)
+	if err != nil {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+
+	// Available is blocks available * fragment size
+	available := int64(statfs.Bavail) * int64(statfs.Bsize)
+
+	// Capacity is total block count * fragment size
+	capacity := int64(statfs.Blocks) * int64(statfs.Bsize)
+
+	// Usage is block being used * fragment size (aka block size).
+	usage := (int64(statfs.Blocks) - int64(statfs.Bfree)) * int64(statfs.Bsize)
+
+	inodes := int64(statfs.Files)
+	inodesFree := int64(statfs.Ffree)
+	inodesUsed := inodes - inodesFree
+
+	return available, capacity, usage, inodes, inodesFree, inodesUsed, nil
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	// Uses the same niceness level as cadvisor.fs does when running du
+	// Uses -B 1 to always scale to a blocksize of 1 byte
+	out, err := exec.Command("nice", "-n", "19", "du", "-s", "-B", "1", path).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("failed command 'du' ($ nice -n 19 du -s -B 1) on path %s with error %v", path, err)
+	}
+	used, err := resource.ParseQuantity(strings.Fields(string(out))[0])
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse 'du' output %s due to error %v", out, err)
+	}
+	used.Format = resource.BinarySI
+	return &used, nil
+}
+
+// Find uses the equivalent of the command `find <path> -dev -printf '.' | wc -c` to count files and directories.
+// While this is not an exact measure of inodes used, it is a very good approximation.
+func Find(path string) (int64, error) {
+	if path == "" {
+		return 0, fmt.Errorf("invalid directory")
+	}
+	var counter byteCounter
+	var stderr bytes.Buffer
+	findCmd := exec.Command("find", path, "-xdev", "-printf", ".")
+	findCmd.Stdout, findCmd.Stderr = &counter, &stderr
+	if err := findCmd.Start(); err != nil {
+		return 0, fmt.Errorf("failed to exec cmd %v - %v; stderr: %v", findCmd.Args, err, stderr.String())
+	}
+	if err := findCmd.Wait(); err != nil {
+		return 0, fmt.Errorf("cmd %v failed. stderr: %s; err: %v", findCmd.Args, stderr.String(), err)
+	}
+	return counter.bytesWritten, nil
+}
+
+// Simple io.Writer implementation that counts how many bytes were written.
+type byteCounter struct{ bytesWritten int64 }
+
+func (b *byteCounter) Write(p []byte) (int, error) {
+	b.bytesWritten += int64(len(p))
+	return len(p), nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_unsupported.go

@@ -0,0 +1,39 @@
+// +build !linux,!darwin,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// FSInfo unsupported returns 0 values for available and capacity and an error.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	return 0, 0, 0, 0, 0, 0, fmt.Errorf("FsInfo not supported for this build.")
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	return nil, fmt.Errorf("Du not supported for this build.")
+}
+
+func Find(path string) (int64, error) {
+	return 0, fmt.Errorf("Find not supported for this build.")
+}

vendor/k8s.io/kubernetes/pkg/volume/util/fs/fs_windows.go

@@ -0,0 +1,77 @@
+// +build windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fs
+
+import (
+	"fmt"
+	"syscall"
+	"unsafe"
+
+	"golang.org/x/sys/windows"
+
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var (
+	modkernel32            = windows.NewLazySystemDLL("kernel32.dll")
+	procGetDiskFreeSpaceEx = modkernel32.NewProc("GetDiskFreeSpaceExW")
+)
+
+// FSInfo returns (available bytes, byte capacity, byte usage, total inodes, inodes free, inode usage, error)
+// for the filesystem that path resides upon.
+func FsInfo(path string) (int64, int64, int64, int64, int64, int64, error) {
+	var freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes int64
+	var err error
+
+	ret, _, err := syscall.Syscall6(
+		procGetDiskFreeSpaceEx.Addr(),
+		4,
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&freeBytesAvailable)),
+		uintptr(unsafe.Pointer(&totalNumberOfBytes)),
+		uintptr(unsafe.Pointer(&totalNumberOfFreeBytes)),
+		0,
+		0,
+	)
+	if ret == 0 {
+		return 0, 0, 0, 0, 0, 0, err
+	}
+
+	return freeBytesAvailable, totalNumberOfBytes, totalNumberOfBytes - freeBytesAvailable, 0, 0, 0, nil
+}
+
+// DiskUsage gets disk usage of specified path.
+func DiskUsage(path string) (*resource.Quantity, error) {
+	_, _, usage, _, _, _, err := FsInfo(path)
+	if err != nil {
+		return nil, err
+	}
+
+	used, err := resource.ParseQuantity(fmt.Sprintf("%d", usage))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse fs usage %d due to %v", usage, err)
+	}
+	used.Format = resource.BinarySI
+	return &used, nil
+}
+
+// Always return zero since inodes is not supported on Windows.
+func Find(path string) (int64, error) {
+	return 0, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/io_util.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// IoUtil is a mockable util for common IO operations
+type IoUtil interface {
+	ReadFile(filename string) ([]byte, error)
+	ReadDir(dirname string) ([]os.FileInfo, error)
+	Lstat(name string) (os.FileInfo, error)
+	EvalSymlinks(path string) (string, error)
+}
+
+type osIOHandler struct{}
+
+//NewIOHandler Create a new IoHandler implementation
+func NewIOHandler() IoUtil {
+	return &osIOHandler{}
+}
+
+func (handler *osIOHandler) ReadFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}
+func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
+	return ioutil.ReadDir(dirname)
+}
+func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
+	return os.Lstat(name)
+}
+func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
+	return filepath.EvalSymlinks(path)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/metrics.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+var storageOperationMetric = prometheus.NewHistogramVec(
+	prometheus.HistogramOpts{
+		Name:    "storage_operation_duration_seconds",
+		Help:    "Storage operation duration",
+		Buckets: []float64{.1, .25, .5, 1, 2.5, 5, 10, 15, 25, 50},
+	},
+	[]string{"volume_plugin", "operation_name"},
+)
+
+var storageOperationErrorMetric = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Name: "storage_operation_errors_total",
+		Help: "Storage operation errors",
+	},
+	[]string{"volume_plugin", "operation_name"},
+)
+
+func init() {
+	registerMetrics()
+}
+
+func registerMetrics() {
+	prometheus.MustRegister(storageOperationMetric)
+	prometheus.MustRegister(storageOperationErrorMetric)
+}
+
+// OperationCompleteHook returns a hook to call when an operation is completed
+func OperationCompleteHook(plugin, operationName string) func(*error) {
+	requestTime := time.Now()
+	opComplete := func(err *error) {
+		timeTaken := time.Since(requestTime).Seconds()
+		// Create metric with operation name and plugin name
+		if *err != nil {
+			storageOperationErrorMetric.WithLabelValues(plugin, operationName).Inc()
+		} else {
+			storageOperationMetric.WithLabelValues(plugin, operationName).Observe(timeTaken)
+		}
+	}
+	return opComplete
+}
+
+// GetFullQualifiedPluginNameForVolume returns full qualified plugin name for
+// given volume. For CSI plugin, it appends plugin driver name at the end of
+// plugin name, e.g. kubernetes.io/csi:csi-hostpath. It helps to distinguish
+// between metrics emitted for CSI volumes which may be handled by different
+// CSI plugin drivers.
+func GetFullQualifiedPluginNameForVolume(pluginName string, spec *volume.Spec) string {
+	if spec != nil && spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil {
+		return fmt.Sprintf("%s:%s", pluginName, spec.PersistentVolume.Spec.CSI.Driver)
+	}
+	return pluginName
+}

vendor/k8s.io/kubernetes/pkg/volume/util/nested_volumes.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"k8s.io/api/core/v1"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// getNestedMountpoints returns a list of mountpoint directories that should be created
+// for the volume indicated by name.
+// note: the returned list is relative to baseDir
+func getNestedMountpoints(name, baseDir string, pod v1.Pod) ([]string, error) {
+	var retval []string
+	checkContainer := func(container *v1.Container) error {
+		var allMountPoints []string // all mount points in this container
+		var myMountPoints []string  // mount points that match name
+		for _, vol := range container.VolumeMounts {
+			cleaned := filepath.Clean(vol.MountPath)
+			allMountPoints = append(allMountPoints, cleaned)
+			if vol.Name == name {
+				myMountPoints = append(myMountPoints, cleaned)
+			}
+		}
+		sort.Strings(allMountPoints)
+		parentPrefix := ".." + string(os.PathSeparator)
+		// Examine each place where this volume is mounted
+		for _, myMountPoint := range myMountPoints {
+			if strings.HasPrefix(myMountPoint, parentPrefix) {
+				// Don't let a container trick us into creating directories outside of its rootfs
+				return fmt.Errorf("Invalid container mount point %v", myMountPoint)
+			}
+			myMPSlash := myMountPoint + string(os.PathSeparator)
+			// The previously found nested mountpoint (or "" if none found yet)
+			prevNestedMP := ""
+			// examine each mount point to see if it's nested beneath this volume
+			// (but skip any that are double-nested beneath this volume)
+			// For example, if this volume is mounted as /dir and other volumes are mounted
+			//              as /dir/nested and /dir/nested/other, only create /dir/nested.
+			for _, mp := range allMountPoints {
+				if !strings.HasPrefix(mp, myMPSlash) {
+					continue // skip -- not nested beneath myMountPoint
+				}
+				if prevNestedMP != "" && strings.HasPrefix(mp, prevNestedMP) {
+					continue // skip -- double nested beneath myMountPoint
+				}
+				// since this mount point is nested, remember it so that we can check that following ones aren't nested beneath this one
+				prevNestedMP = mp + string(os.PathSeparator)
+				retval = append(retval, mp[len(myMPSlash):])
+			}
+		}
+		return nil
+	}
+	for _, container := range pod.Spec.InitContainers {
+		if err := checkContainer(&container); err != nil {
+			return nil, err
+		}
+	}
+	for _, container := range pod.Spec.Containers {
+		if err := checkContainer(&container); err != nil {
+			return nil, err
+		}
+	}
+	return retval, nil
+}
+
+// MakeNestedMountpoints creates mount points in baseDir for volumes mounted beneath name
+func MakeNestedMountpoints(name, baseDir string, pod v1.Pod) error {
+	dirs, err := getNestedMountpoints(name, baseDir, pod)
+	if err != nil {
+		return err
+	}
+	for _, dir := range dirs {
+		err := os.MkdirAll(path.Join(baseDir, dir), 0755)
+		if err != nil {
+			return fmt.Errorf("Unable to create nested volume mountpoints: %v", err)
+		}
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/recyclerclient/recycler_client.go

@@ -0,0 +1,267 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package recyclerclient
+
+import (
+	"fmt"
+	"sync"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/watch"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
+)
+
+type RecycleEventRecorder func(eventtype, message string)
+
+// RecycleVolumeByWatchingPodUntilCompletion is intended for use with volume
+// Recyclers. This function will save the given Pod to the API and watch it
+// until it completes, fails, or the pod's ActiveDeadlineSeconds is exceeded,
+// whichever comes first. An attempt to delete a recycler pod is always
+// attempted before returning.
+//
+// In case there is a pod with the same namespace+name already running, this
+// function deletes it as it is not able to judge if it is an old recycler
+// or user has forged a fake recycler to block Kubernetes from recycling.//
+//
+//  pod - the pod designed by a volume plugin to recycle the volume. pod.Name
+//        will be overwritten with unique name based on PV.Name.
+//	client - kube client for API operations.
+func RecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, kubeClient clientset.Interface, recorder RecycleEventRecorder) error {
+	return internalRecycleVolumeByWatchingPodUntilCompletion(pvName, pod, newRecyclerClient(kubeClient, recorder))
+}
+
+// same as above func comments, except 'recyclerClient' is a narrower pod API
+// interface to ease testing
+func internalRecycleVolumeByWatchingPodUntilCompletion(pvName string, pod *v1.Pod, recyclerClient recyclerClient) error {
+	klog.V(5).Infof("creating recycler pod for volume %s\n", pod.Name)
+
+	// Generate unique name for the recycler pod - we need to get "already
+	// exists" error when a previous controller has already started recycling
+	// the volume. Here we assume that pv.Name is already unique.
+	pod.Name = "recycler-for-" + pvName
+	pod.GenerateName = ""
+
+	stopChannel := make(chan struct{})
+	defer close(stopChannel)
+	podCh, err := recyclerClient.WatchPod(pod.Name, pod.Namespace, stopChannel)
+	if err != nil {
+		klog.V(4).Infof("cannot start watcher for pod %s/%s: %v", pod.Namespace, pod.Name, err)
+		return err
+	}
+
+	// Start the pod
+	_, err = recyclerClient.CreatePod(pod)
+	if err != nil {
+		if errors.IsAlreadyExists(err) {
+			deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace)
+			if deleteErr != nil {
+				return fmt.Errorf("failed to delete old recycler pod %s/%s: %s", pod.Namespace, pod.Name, deleteErr)
+			}
+			// Recycler will try again and the old pod will be hopefully deleted
+			// at that time.
+			return fmt.Errorf("old recycler pod found, will retry later")
+		}
+		return fmt.Errorf("unexpected error creating recycler pod:  %+v", err)
+	}
+	err = waitForPod(pod, recyclerClient, podCh)
+
+	// In all cases delete the recycler pod and log its result.
+	klog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name)
+	deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace)
+	if deleteErr != nil {
+		klog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err)
+	}
+
+	// Returning recycler error is preferred, the pod will be deleted again on
+	// the next retry.
+	if err != nil {
+		return fmt.Errorf("failed to recycle volume: %s", err)
+	}
+
+	// Recycle succeeded but we failed to delete the recycler pod. Report it,
+	// the controller will re-try recycling the PV again shortly.
+	if deleteErr != nil {
+		return fmt.Errorf("failed to delete recycler pod: %s", deleteErr)
+	}
+
+	return nil
+}
+
+// waitForPod watches the pod it until it finishes and send all events on the
+// pod to the PV.
+func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.Event) error {
+	for {
+		event, ok := <-podCh
+		if !ok {
+			return fmt.Errorf("recycler pod %q watch channel had been closed", pod.Name)
+		}
+		switch event.Object.(type) {
+		case *v1.Pod:
+			// POD changed
+			pod := event.Object.(*v1.Pod)
+			klog.V(4).Infof("recycler pod update received: %s %s/%s %s", event.Type, pod.Namespace, pod.Name, pod.Status.Phase)
+			switch event.Type {
+			case watch.Added, watch.Modified:
+				if pod.Status.Phase == v1.PodSucceeded {
+					// Recycle succeeded.
+					return nil
+				}
+				if pod.Status.Phase == v1.PodFailed {
+					if pod.Status.Message != "" {
+						return fmt.Errorf(pod.Status.Message)
+					} else {
+						return fmt.Errorf("pod failed, pod.Status.Message unknown.")
+					}
+				}
+
+			case watch.Deleted:
+				return fmt.Errorf("recycler pod was deleted")
+
+			case watch.Error:
+				return fmt.Errorf("recycler pod watcher failed")
+			}
+
+		case *v1.Event:
+			// Event received
+			podEvent := event.Object.(*v1.Event)
+			klog.V(4).Infof("recycler event received: %s %s/%s %s/%s %s", event.Type, podEvent.Namespace, podEvent.Name, podEvent.InvolvedObject.Namespace, podEvent.InvolvedObject.Name, podEvent.Message)
+			if event.Type == watch.Added {
+				recyclerClient.Event(podEvent.Type, podEvent.Message)
+			}
+		}
+	}
+}
+
+// recyclerClient abstracts access to a Pod by providing a narrower interface.
+// This makes it easier to mock a client for testing.
+type recyclerClient interface {
+	CreatePod(pod *v1.Pod) (*v1.Pod, error)
+	GetPod(name, namespace string) (*v1.Pod, error)
+	DeletePod(name, namespace string) error
+	// WatchPod returns a ListWatch for watching a pod.  The stopChannel is used
+	// to close the reflector backing the watch.  The caller is responsible for
+	// derring a close on the channel to stop the reflector.
+	WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error)
+	// Event sends an event to the volume that is being recycled.
+	Event(eventtype, message string)
+}
+
+func newRecyclerClient(client clientset.Interface, recorder RecycleEventRecorder) recyclerClient {
+	return &realRecyclerClient{
+		client,
+		recorder,
+	}
+}
+
+type realRecyclerClient struct {
+	client   clientset.Interface
+	recorder RecycleEventRecorder
+}
+
+func (c *realRecyclerClient) CreatePod(pod *v1.Pod) (*v1.Pod, error) {
+	return c.client.CoreV1().Pods(pod.Namespace).Create(pod)
+}
+
+func (c *realRecyclerClient) GetPod(name, namespace string) (*v1.Pod, error) {
+	return c.client.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{})
+}
+
+func (c *realRecyclerClient) DeletePod(name, namespace string) error {
+	return c.client.CoreV1().Pods(namespace).Delete(name, nil)
+}
+
+func (c *realRecyclerClient) Event(eventtype, message string) {
+	c.recorder(eventtype, message)
+}
+
+// WatchPod watches a pod and events related to it. It sends pod updates and events over the returned channel
+// It will continue until stopChannel is closed
+func (c *realRecyclerClient) WatchPod(name, namespace string, stopChannel chan struct{}) (<-chan watch.Event, error) {
+	podSelector, err := fields.ParseSelector("metadata.name=" + name)
+	if err != nil {
+		return nil, err
+	}
+	options := metav1.ListOptions{
+		FieldSelector: podSelector.String(),
+		Watch:         true,
+	}
+
+	podWatch, err := c.client.CoreV1().Pods(namespace).Watch(options)
+	if err != nil {
+		return nil, err
+	}
+
+	eventSelector, _ := fields.ParseSelector("involvedObject.name=" + name)
+	eventWatch, err := c.client.CoreV1().Events(namespace).Watch(metav1.ListOptions{
+		FieldSelector: eventSelector.String(),
+		Watch:         true,
+	})
+	if err != nil {
+		podWatch.Stop()
+		return nil, err
+	}
+
+	eventCh := make(chan watch.Event, 30)
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	go func() {
+		defer close(eventCh)
+		wg.Wait()
+	}()
+
+	go func() {
+		defer eventWatch.Stop()
+		defer wg.Done()
+		for {
+			select {
+			case _ = <-stopChannel:
+				return
+			case eventEvent, ok := <-eventWatch.ResultChan():
+				if !ok {
+					return
+				} else {
+					eventCh <- eventEvent
+				}
+			}
+		}
+	}()
+
+	go func() {
+		defer podWatch.Stop()
+		defer wg.Done()
+		for {
+			select {
+			case <-stopChannel:
+				return
+
+			case podEvent, ok := <-podWatch.ResultChan():
+				if !ok {
+					return
+				} else {
+					eventCh <- podEvent
+				}
+			}
+		}
+	}()
+
+	return eventCh, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/resize_util.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/strategicpatch"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/resizefs"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+var (
+	knownResizeConditions = map[v1.PersistentVolumeClaimConditionType]bool{
+		v1.PersistentVolumeClaimFileSystemResizePending: true,
+		v1.PersistentVolumeClaimResizing:                true,
+	}
+)
+
+type resizeProcessStatus struct {
+	condition v1.PersistentVolumeClaimCondition
+	processed bool
+}
+
+// ClaimToClaimKey return namespace/name string for pvc
+func ClaimToClaimKey(claim *v1.PersistentVolumeClaim) string {
+	return fmt.Sprintf("%s/%s", claim.Namespace, claim.Name)
+}
+
+// MarkFSResizeFinished marks file system resizing as done
+func MarkFSResizeFinished(
+	pvc *v1.PersistentVolumeClaim,
+	capacity v1.ResourceList,
+	kubeClient clientset.Interface) error {
+	newPVC := pvc.DeepCopy()
+	newPVC.Status.Capacity = capacity
+	newPVC = MergeResizeConditionOnPVC(newPVC, []v1.PersistentVolumeClaimCondition{})
+	_, err := PatchPVCStatus(pvc /*oldPVC*/, newPVC, kubeClient)
+	return err
+}
+
+// PatchPVCStatus updates PVC status using PATCH verb
+func PatchPVCStatus(
+	oldPVC *v1.PersistentVolumeClaim,
+	newPVC *v1.PersistentVolumeClaim,
+	kubeClient clientset.Interface) (*v1.PersistentVolumeClaim, error) {
+	pvcName := oldPVC.Name
+
+	oldData, err := json.Marshal(oldPVC)
+	if err != nil {
+		return nil, fmt.Errorf("PatchPVCStatus.Failed to marshal oldData for pvc %q with %v", pvcName, err)
+	}
+
+	newData, err := json.Marshal(newPVC)
+	if err != nil {
+		return nil, fmt.Errorf("PatchPVCStatus.Failed to marshal newData for pvc %q with %v", pvcName, err)
+	}
+
+	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, oldPVC)
+	if err != nil {
+		return nil, fmt.Errorf("PatchPVCStatus.Failed to CreateTwoWayMergePatch for pvc %q with %v ", pvcName, err)
+	}
+	updatedClaim, updateErr := kubeClient.CoreV1().PersistentVolumeClaims(oldPVC.Namespace).
+		Patch(pvcName, types.StrategicMergePatchType, patchBytes, "status")
+	if updateErr != nil {
+		return nil, fmt.Errorf("PatchPVCStatus.Failed to patch PVC %q with %v", pvcName, updateErr)
+	}
+	return updatedClaim, nil
+}
+
+// MergeResizeConditionOnPVC updates pvc with requested resize conditions
+// leaving other conditions untouched.
+func MergeResizeConditionOnPVC(
+	pvc *v1.PersistentVolumeClaim,
+	resizeConditions []v1.PersistentVolumeClaimCondition) *v1.PersistentVolumeClaim {
+	resizeConditionMap := map[v1.PersistentVolumeClaimConditionType]*resizeProcessStatus{}
+
+	for _, condition := range resizeConditions {
+		resizeConditionMap[condition.Type] = &resizeProcessStatus{condition, false}
+	}
+
+	oldConditions := pvc.Status.Conditions
+	newConditions := []v1.PersistentVolumeClaimCondition{}
+	for _, condition := range oldConditions {
+		// If Condition is of not resize type, we keep it.
+		if _, ok := knownResizeConditions[condition.Type]; !ok {
+			newConditions = append(newConditions, condition)
+			continue
+		}
+
+		if newCondition, ok := resizeConditionMap[condition.Type]; ok {
+			if newCondition.condition.Status != condition.Status {
+				newConditions = append(newConditions, newCondition.condition)
+			} else {
+				newConditions = append(newConditions, condition)
+			}
+			newCondition.processed = true
+		}
+	}
+
+	// append all unprocessed conditions
+	for _, newCondition := range resizeConditionMap {
+		if !newCondition.processed {
+			newConditions = append(newConditions, newCondition.condition)
+		}
+	}
+	pvc.Status.Conditions = newConditions
+	return pvc
+}
+
+// GenericResizeFS : call generic filesystem resizer for plugins that don't have any special filesystem resize requirements
+func GenericResizeFS(host volume.VolumeHost, pluginName, devicePath, deviceMountPath string) (bool, error) {
+	mounter := host.GetMounter(pluginName)
+	diskFormatter := &mount.SafeFormatAndMount{
+		Interface: mounter,
+		Exec:      host.GetExec(pluginName),
+	}
+	resizer := resizefs.NewResizeFs(diskFormatter)
+	return resizer.Resize(devicePath, deviceMountPath)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import "os"
+
+// Interface defines the set of methods all subpathers must implement
+type Interface interface {
+	// CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given
+	// pod volume directory.
+	CleanSubPaths(poodDir string, volumeName string) error
+
+	// PrepareSafeSubpath does everything that's necessary to prepare a subPath
+	// that's 1) inside given volumePath and 2) immutable after this call.
+	//
+	// newHostPath - location of prepared subPath. It should be used instead of
+	// hostName when running the container.
+	// cleanupAction - action to run when the container is running or it failed to start.
+	//
+	// CleanupAction must be called immediately after the container with given
+	// subpath starts. On the other hand, Interface.CleanSubPaths must be called
+	// when the pod finishes.
+	PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error)
+
+	// SafeMakeDir creates subdir within given base. It makes sure that the
+	// created directory does not escape given base directory mis-using
+	// symlinks. Note that the function makes sure that it creates the directory
+	// somewhere under the base, nothing else. E.g. if the directory already
+	// exists, it may exist outside of the base due to symlinks.
+	// This method should be used if the directory to create is inside volume
+	// that's under user control. User must not be able to use symlinks to
+	// escape the volume to create directories somewhere else.
+	SafeMakeDir(subdir string, base string, perm os.FileMode) error
+}
+
+// Subpath defines the attributes of a subpath
+type Subpath struct {
+	// index of the VolumeMount for this container
+	VolumeMountIndex int
+
+	// Full path to the subpath directory on the host
+	Path string
+
+	// name of the volume that is a valid directory name.
+	VolumeName string
+
+	// Full path to the volume path
+	VolumePath string
+
+	// Path to the pod's directory, including pod UID
+	PodDir string
+
+	// Name of the container
+	ContainerName string
+}
+
+// Compile time-check for all implementers of subpath interface
+var _ Interface = &subpath{}
+var _ Interface = &FakeSubpath{}
+
+// FakeSubpath is a subpather implementation for testing
+type FakeSubpath struct{}
+
+// PrepareSafeSubpath is a fake implementation of PrepareSafeSubpath. Always returns
+// newHostPath == subPath.Path
+func (fs *FakeSubpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, nil
+}
+
+// CleanSubPaths is a fake implementation of CleanSubPaths. It is a noop
+func (fs *FakeSubpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir is a fake implementation of SafeMakeDir. It is a noop
+func (fs *FakeSubpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go

@@ -0,0 +1,563 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+const (
+	// place for subpath mounts
+	// TODO: pass in directory using kubelet_getters instead
+	containerSubPathDirectoryName = "volume-subpaths"
+	// syscall.Openat flags used to traverse directories not following symlinks
+	nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW
+	// flags for getting file descriptor without following the symlink
+	openFDFlags = unix.O_NOFOLLOW | unix.O_PATH
+)
+
+type subpath struct {
+	mounter mount.Interface
+}
+
+// New returns a subpath.Interface for the current system
+func New(mounter mount.Interface) Interface {
+	return &subpath{
+		mounter: mounter,
+	}
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(sp.mounter, podDir, volumeName)
+}
+
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	newHostPath, err = doBindSubPath(sp.mounter, subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+// This implementation is shared between Linux and NsEnter
+func safeOpenSubPath(mounter mount.Interface, subpath Subpath) (int, error) {
+	if !mount.PathWithinBase(subpath.Path, subpath.VolumePath) {
+		return -1, fmt.Errorf("subpath %q not within volume path %q", subpath.Path, subpath.VolumePath)
+	}
+	fd, err := doSafeOpen(subpath.Path, subpath.VolumePath)
+	if err != nil {
+		return -1, fmt.Errorf("error opening subpath %v: %v", subpath.Path, err)
+	}
+	return fd, nil
+}
+
+// prepareSubpathTarget creates target for bind-mount of subpath. It returns
+// "true" when the target already exists and something is mounted there.
+// Given Subpath must have all paths with already resolved symlinks and with
+// paths relevant to kubelet (when it runs in a container).
+// This function is called also by NsEnterMounter. It works because
+// /var/lib/kubelet is mounted from the host into the container with Kubelet as
+// /var/lib/kubelet too.
+func prepareSubpathTarget(mounter mount.Interface, subpath Subpath) (bool, string, error) {
+	// Early check for already bind-mounted subpath.
+	bindPathTarget := getSubpathBindTarget(subpath)
+	notMount, err := mounter.IsNotMountPoint(bindPathTarget)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, "", fmt.Errorf("error checking path %s for mount: %s", bindPathTarget, err)
+		}
+		// Ignore ErrorNotExist: the file/directory will be created below if it does not exist yet.
+		notMount = true
+	}
+	if !notMount {
+		// It's already mounted
+		klog.V(5).Infof("Skipping bind-mounting subpath %s: already mounted", bindPathTarget)
+		return true, bindPathTarget, nil
+	}
+
+	// bindPathTarget is in /var/lib/kubelet and thus reachable without any
+	// translation even to containerized kubelet.
+	bindParent := filepath.Dir(bindPathTarget)
+	err = os.MkdirAll(bindParent, 0750)
+	if err != nil && !os.IsExist(err) {
+		return false, "", fmt.Errorf("error creating directory %s: %s", bindParent, err)
+	}
+
+	t, err := os.Lstat(subpath.Path)
+	if err != nil {
+		return false, "", fmt.Errorf("lstat %s failed: %s", subpath.Path, err)
+	}
+
+	if t.Mode()&os.ModeDir > 0 {
+		if err = os.Mkdir(bindPathTarget, 0750); err != nil && !os.IsExist(err) {
+			return false, "", fmt.Errorf("error creating directory %s: %s", bindPathTarget, err)
+		}
+	} else {
+		// "/bin/touch <bindPathTarget>".
+		// A file is enough for all possible targets (symlink, device, pipe,
+		// socket, ...), bind-mounting them into a file correctly changes type
+		// of the target file.
+		if err = ioutil.WriteFile(bindPathTarget, []byte{}, 0640); err != nil {
+			return false, "", fmt.Errorf("error creating file %s: %s", bindPathTarget, err)
+		}
+	}
+	return false, bindPathTarget, nil
+}
+
+func getSubpathBindTarget(subpath Subpath) string {
+	// containerName is DNS label, i.e. safe as a directory name.
+	return filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName, strconv.Itoa(subpath.VolumeMountIndex))
+}
+
+func doBindSubPath(mounter mount.Interface, subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs on the host:
+	// - safely open the subpath
+	// - bind-mount /proc/<pid of kubelet>/fd/<fd> to subpath target
+	// User can't change /proc/<pid of kubelet>/fd/<fd> to point to a bad place.
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	newVolumePath, err := filepath.EvalSymlinks(subpath.VolumePath)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	newPath, err := filepath.EvalSymlinks(subpath.Path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, newPath, subpath.VolumePath)
+	subpath.VolumePath = newVolumePath
+	subpath.Path = newPath
+
+	fd, err := safeOpenSubPath(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			klog.V(4).Infof("doBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(mounter, subpath); cleanErr != nil {
+				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	kubeletPid := os.Getpid()
+	mountSource := fmt.Sprintf("/proc/%d/fd/%v", kubeletPid, fd)
+
+	// Do the bind mount
+	options := []string{"bind"}
+	klog.V(5).Infof("bind mounting %q at %q", mountSource, bindPathTarget)
+	if err = mounter.Mount(mountSource, bindPathTarget, "" /*fstype*/, options); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", subpath.Path, err)
+	}
+	success = true
+
+	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// This implementation is shared between Linux and NsEnter
+func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string) error {
+	// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/*
+	subPathDir := filepath.Join(podDir, containerSubPathDirectoryName, volumeName)
+	klog.V(4).Infof("Cleaning up subpath mounts for %s", subPathDir)
+
+	containerDirs, err := ioutil.ReadDir(subPathDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil
+		}
+		return fmt.Errorf("error reading %s: %s", subPathDir, err)
+	}
+
+	for _, containerDir := range containerDirs {
+		if !containerDir.IsDir() {
+			klog.V(4).Infof("Container file is not a directory: %s", containerDir.Name())
+			continue
+		}
+		klog.V(4).Infof("Cleaning up subpath mounts for container %s", containerDir.Name())
+
+		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
+		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
+		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
+			if path == fullContainerDirPath {
+				// Skip top level directory
+				return nil
+			}
+
+			// pass through errors and let doCleanSubPath handle them
+			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
+				return err
+			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
+		}
+
+		// Whole container has been processed, remove its directory.
+		if err := os.Remove(fullContainerDirPath); err != nil {
+			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
+		}
+		klog.V(5).Infof("Removed %s", fullContainerDirPath)
+	}
+	// Whole pod volume subpaths have been cleaned up, remove its subpath directory.
+	if err := os.Remove(subPathDir); err != nil {
+		return fmt.Errorf("error deleting %s: %s", subPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", subPathDir)
+
+	// Remove entire subpath directory if it's the last one
+	podSubPathDir := filepath.Join(podDir, containerSubPathDirectoryName)
+	if err := os.Remove(podSubPathDir); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("error deleting %s: %s", podSubPathDir, err)
+	}
+	klog.V(5).Infof("Removed %s", podSubPathDir)
+	return nil
+}
+
+// doCleanSubPath tears down the single subpath bind mount
+func doCleanSubPath(mounter mount.Interface, fullContainerDirPath, subPathIndex string) error {
+	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
+	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
+	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
+
+	if err := mount.CleanupMountPoint(fullSubPath, mounter, true); err != nil {
+		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
+	}
+
+	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
+	return nil
+}
+
+// cleanSubPath will teardown the subpath bind mount and any remove any directories if empty
+func cleanSubPath(mounter mount.Interface, subpath Subpath) error {
+	containerDir := filepath.Join(subpath.PodDir, containerSubPathDirectoryName, subpath.VolumeName, subpath.ContainerName)
+
+	// Clean subdir bindmount
+	if err := doCleanSubPath(mounter, containerDir, strconv.Itoa(subpath.VolumeMountIndex)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	// Recusively remove directories if empty
+	if err := removeEmptyDirs(subpath.PodDir, containerDir); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// removeEmptyDirs works backwards from endDir to baseDir and removes each directory
+// if it is empty.  It stops once it encounters a directory that has content
+func removeEmptyDirs(baseDir, endDir string) error {
+	if !mount.PathWithinBase(endDir, baseDir) {
+		return fmt.Errorf("endDir %q is not within baseDir %q", endDir, baseDir)
+	}
+
+	for curDir := endDir; curDir != baseDir; curDir = filepath.Dir(curDir) {
+		s, err := os.Stat(curDir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				klog.V(5).Infof("curDir %q doesn't exist, skipping", curDir)
+				continue
+			}
+			return fmt.Errorf("error stat %q: %v", curDir, err)
+		}
+		if !s.IsDir() {
+			return fmt.Errorf("path %q not a directory", curDir)
+		}
+
+		err = os.Remove(curDir)
+		if os.IsExist(err) {
+			klog.V(5).Infof("Directory %q not empty, not removing", curDir)
+			break
+		} else if err != nil {
+			return fmt.Errorf("error removing directory %q: %v", curDir, err)
+		}
+		klog.V(5).Infof("Removed directory %q", curDir)
+	}
+	return nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter. Both pathname
+// and base must be either already resolved symlinks or thet will be resolved in
+// kubelet's mount namespace (in case it runs containerized).
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", existingPath, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	parentFD, err := doSafeOpen(fullExistingPath, base)
+	if err != nil {
+		return fmt.Errorf("cannot open directory %s: %s", existingPath, err)
+	}
+	childFD := -1
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+			}
+		}
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink.
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		err = syscall.Mkdirat(parentFD, currentPath, uint32(perm))
+		if err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		// Dive into the created directory
+		childFD, err := syscall.Openat(parentFD, dir, nofollowFlags, 0)
+		if err != nil {
+			return fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+		// We can be sure that childFD is safe to use. It could be changed
+		// by user after Mkdirat() and before Openat(), however:
+		// - it could not be changed to symlink - we use nofollowFlags
+		// - it could be changed to a file (or device, pipe, socket, ...)
+		//   but either subsequent Mkdirat() fails or we mount this file
+		//   to user's container. Security is no violated in both cases
+		//   and user either gets error or the file that it can already access.
+
+		if err = syscall.Close(parentFD); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for safemkdir(%v): %v", parentFD, pathname, err)
+		}
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// Everything was created. mkdirat(..., perm) above was affected by current
+	// umask and we must apply the right permissions to the last directory
+	// (that's the one that will be available to the container as subpath)
+	// so user can read/write it. This is the behavior of previous code.
+	// TODO: chmod all created directories, not just the last one.
+	// parentFD is the last created directory.
+
+	// Translate perm (os.FileMode) to uint32 that fchmod() expects
+	kernelPerm := uint32(perm & os.ModePerm)
+	if perm&os.ModeSetgid > 0 {
+		kernelPerm |= syscall.S_ISGID
+	}
+	if perm&os.ModeSetuid > 0 {
+		kernelPerm |= syscall.S_ISUID
+	}
+	if perm&os.ModeSticky > 0 {
+		kernelPerm |= syscall.S_ISVTX
+	}
+	if err = syscall.Fchmod(parentFD, kernelPerm); err != nil {
+		return fmt.Errorf("chmod %q failed: %s", currentPath, err)
+	}
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	// Do OpenAt in a loop to find the first non-existing dir. Resolve symlinks.
+	// This should be faster than looping through all dirs and calling os.Stat()
+	// on each of them, as the symlinks are resolved only once with OpenAt().
+	currentPath := base
+	fd, err := syscall.Open(currentPath, syscall.O_RDONLY, 0)
+	if err != nil {
+		return pathname, nil, fmt.Errorf("error opening %s: %s", currentPath, err)
+	}
+	defer func() {
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+	}()
+	for i, dir := range dirs {
+		// Using O_PATH here will prevent hangs in case user replaces directory with
+		// fifo
+		childFD, err := syscall.Openat(fd, dir, unix.O_PATH, 0)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return currentPath, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+		if err = syscall.Close(fd); err != nil {
+			klog.V(4).Infof("Closing FD %v failed for findExistingPrefix(%v): %v", fd, pathname, err)
+		}
+		fd = childFD
+		currentPath = filepath.Join(currentPath, dir)
+	}
+	return pathname, []string{}, nil
+}
+
+// This implementation is shared between Linux and NsEnterMounter
+// Open path and return its fd.
+// Symlinks are disallowed (pathname must already resolve symlinks),
+// and the path must be within the base directory.
+func doSafeOpen(pathname string, base string) (int, error) {
+	pathname = filepath.Clean(pathname)
+	base = filepath.Clean(base)
+
+	// Calculate segments to follow
+	subpath, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return -1, err
+	}
+	segments := strings.Split(subpath, string(filepath.Separator))
+
+	// Assumption: base is the only directory that we have under control.
+	// Base dir is not allowed to be a symlink.
+	parentFD, err := syscall.Open(base, nofollowFlags, 0)
+	if err != nil {
+		return -1, fmt.Errorf("cannot open directory %s: %s", base, err)
+	}
+	defer func() {
+		if parentFD != -1 {
+			if err = syscall.Close(parentFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", parentFD, pathname, err)
+			}
+		}
+	}()
+
+	childFD := -1
+	defer func() {
+		if childFD != -1 {
+			if err = syscall.Close(childFD); err != nil {
+				klog.V(4).Infof("Closing FD %v failed for safeopen(%v): %v", childFD, pathname, err)
+			}
+		}
+	}()
+
+	currentPath := base
+
+	// Follow the segments one by one using openat() to make
+	// sure the user cannot change already existing directories into symlinks.
+	for _, seg := range segments {
+		currentPath = filepath.Join(currentPath, seg)
+		if !mount.PathWithinBase(currentPath, base) {
+			return -1, fmt.Errorf("path %s is outside of allowed base %s", currentPath, base)
+		}
+
+		klog.V(5).Infof("Opening path %s", currentPath)
+		childFD, err = syscall.Openat(parentFD, seg, openFDFlags, 0)
+		if err != nil {
+			return -1, fmt.Errorf("cannot open %s: %s", currentPath, err)
+		}
+
+		var deviceStat unix.Stat_t
+		err := unix.Fstat(childFD, &deviceStat)
+		if err != nil {
+			return -1, fmt.Errorf("Error running fstat on %s with %v", currentPath, err)
+		}
+		fileFmt := deviceStat.Mode & syscall.S_IFMT
+		if fileFmt == syscall.S_IFLNK {
+			return -1, fmt.Errorf("Unexpected symlink found %s", currentPath)
+		}
+
+		// Close parentFD
+		if err = syscall.Close(parentFD); err != nil {
+			return -1, fmt.Errorf("closing fd for %q failed: %v", filepath.Dir(currentPath), err)
+		}
+		// Set child to new parent
+		parentFD = childFD
+		childFD = -1
+	}
+
+	// We made it to the end, return this fd, don't close it
+	finalFD := parentFD
+	parentFD = -1
+
+	return finalFD, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_nsenter.go

@@ -0,0 +1,186 @@
+// +build linux
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog"
+	"k8s.io/utils/nsenter"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+)
+
+type subpathNSE struct {
+	mounter mount.Interface
+	ne      *nsenter.Nsenter
+	rootDir string
+}
+
+// Compile time-check for all implementers of subpath interface
+var _ Interface = &subpathNSE{}
+
+// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
+// It is only valid on Linux systems
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return &subpathNSE{
+		mounter: mounter,
+		ne:      ne,
+		rootDir: rootDir,
+	}
+}
+
+func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
+	return doCleanSubPaths(sp.mounter, podDir, volumeName)
+}
+
+func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	// Bind-mount the subpath to avoid using symlinks in subpaths.
+	newHostPath, err = sp.doNsEnterBindSubPath(subPath)
+
+	// There is no action when the container starts. Bind-mount will be cleaned
+	// when container stops by CleanSubPaths.
+	cleanupAction = nil
+	return newHostPath, cleanupAction, err
+}
+
+func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	fullSubdirPath := filepath.Join(base, subdir)
+	evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
+	}
+	evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
+
+	evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+	evaluatedBase = filepath.Clean(evaluatedBase)
+
+	rootDir := filepath.Clean(sp.rootDir)
+	if mount.PathWithinBase(evaluatedBase, rootDir) {
+		// Base is in /var/lib/kubelet. This directory is shared between the
+		// container with kubelet and the host. We don't need to add '/rootfs'.
+		// This is useful when /rootfs is mounted as read-only - we can still
+		// create subpaths for paths in /var/lib/kubelet.
+		return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
+	}
+
+	// Base is somewhere on the host's filesystem. Add /rootfs and try to make
+	// the directory there.
+	// This requires /rootfs to be writable.
+	kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
+	kubeletBase := sp.ne.KubeletPath(evaluatedBase)
+	return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
+}
+
+func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
+	// Linux, kubelet runs in a container:
+	// - safely open the subpath
+	// - bind-mount the subpath to target (this can be unsafe)
+	// - check that we mounted the right thing by comparing device ID and inode
+	//   of the subpath (via safely opened fd) and the target (that's under our
+	//   control)
+
+	// Evaluate all symlinks here once for all subsequent functions.
+	evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
+	}
+	evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
+	if err != nil {
+		return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
+	}
+	klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
+	subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
+	subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
+
+	// Check the subpath is correct and open it
+	fd, err := safeOpenSubPath(sp.mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	defer syscall.Close(fd)
+
+	alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
+	if err != nil {
+		return "", err
+	}
+	if alreadyMounted {
+		return bindPathTarget, nil
+	}
+
+	success := false
+	defer func() {
+		// Cleanup subpath on error
+		if !success {
+			klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
+			if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
+				klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
+			}
+		}
+	}()
+
+	// Leap of faith: optimistically expect that nobody has modified previously
+	// expanded evalSubPath with evil symlinks and bind-mount it.
+	// Mount is done on the host! don't use kubelet path!
+	klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
+	if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
+		return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
+	}
+
+	// Check that the bind-mount target is the same inode and device as the
+	// source that we keept open, i.e. we mounted the right thing.
+	err = checkDeviceInode(fd, bindPathTarget)
+	if err != nil {
+		return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
+	}
+
+	success = true
+	klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
+	return bindPathTarget, nil
+}
+
+// checkDeviceInode checks that opened file and path represent the same file.
+func checkDeviceInode(fd int, path string) error {
+	var srcStat, dstStat unix.Stat_t
+	err := unix.Fstat(fd, &srcStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on subpath FD: %v", err)
+	}
+
+	err = unix.Stat(path, &dstStat)
+	if err != nil {
+		return fmt.Errorf("error running fstat on %s: %v", path, err)
+	}
+
+	if srcStat.Dev != dstStat.Dev {
+		return fmt.Errorf("different device number")
+	}
+	if srcStat.Ino != dstStat.Ino {
+		return fmt.Errorf("different inode")
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_unsupported.go

@@ -0,0 +1,54 @@
+// +build !linux,!windows
+
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"errors"
+	"os"
+
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/nsenter"
+)
+
+type subpath struct{}
+
+var errUnsupported = errors.New("util/subpath on this platform is not supported")
+
+// New returns a subpath.Interface for the current system.
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	return subPath.Path, nil, errUnsupported
+}
+
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return errUnsupported
+}
+
+func (sp *subpath) SafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	return errUnsupported
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_windows.go

@@ -0,0 +1,284 @@
+// +build windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subpath
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/utils/nsenter"
+)
+
+type subpath struct{}
+
+// New returns a subpath.Interface for the current system
+func New(mount.Interface) Interface {
+	return &subpath{}
+}
+
+// NewNSEnter is to satisfy the compiler for having NewSubpathNSEnter exist for all
+// OS choices. however, NSEnter is only valid on Linux
+func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
+	return nil
+}
+
+// check whether hostPath is within volume path
+// this func will lock all intermediate subpath directories, need to close handle outside of this func after container started
+func lockAndCheckSubPath(volumePath, hostPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(hostPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	finalSubPath, err := filepath.EvalSymlinks(hostPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", hostPath, err)
+	}
+	finalVolumePath, err := filepath.EvalSymlinks(volumePath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("cannot read link %s: %s", volumePath, err)
+	}
+
+	return lockAndCheckSubPathWithoutSymlink(finalVolumePath, finalSubPath)
+}
+
+// lock all intermediate subPath directories and check they are all within volumePath
+// volumePath & subPath should not contain any symlink, otherwise it will return error
+func lockAndCheckSubPathWithoutSymlink(volumePath, subPath string) ([]uintptr, error) {
+	if len(volumePath) == 0 || len(subPath) == 0 {
+		return []uintptr{}, nil
+	}
+
+	// get relative path to volumePath
+	relSubPath, err := filepath.Rel(volumePath, subPath)
+	if err != nil {
+		return []uintptr{}, fmt.Errorf("Rel(%s, %s) error: %v", volumePath, subPath, err)
+	}
+	if mount.StartsWithBackstep(relSubPath) {
+		return []uintptr{}, fmt.Errorf("SubPath %q not within volume path %q", subPath, volumePath)
+	}
+
+	if relSubPath == "." {
+		// volumePath and subPath are equal
+		return []uintptr{}, nil
+	}
+
+	fileHandles := []uintptr{}
+	var errorResult error
+
+	currentFullPath := volumePath
+	dirs := strings.Split(relSubPath, string(os.PathSeparator))
+	for _, dir := range dirs {
+		// lock intermediate subPath directory first
+		currentFullPath = filepath.Join(currentFullPath, dir)
+		handle, err := lockPath(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("cannot lock path %s: %s", currentFullPath, err)
+			break
+		}
+		fileHandles = append(fileHandles, handle)
+
+		// make sure intermediate subPath directory does not contain symlink any more
+		stat, err := os.Lstat(currentFullPath)
+		if err != nil {
+			errorResult = fmt.Errorf("Lstat(%q) error: %v", currentFullPath, err)
+			break
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			errorResult = fmt.Errorf("subpath %q is an unexpected symlink after EvalSymlinks", currentFullPath)
+			break
+		}
+
+		if !mount.PathWithinBase(currentFullPath, volumePath) {
+			errorResult = fmt.Errorf("SubPath %q not within volume path %q", currentFullPath, volumePath)
+			break
+		}
+	}
+
+	return fileHandles, errorResult
+}
+
+// unlockPath unlock directories
+func unlockPath(fileHandles []uintptr) {
+	if fileHandles != nil {
+		for _, handle := range fileHandles {
+			syscall.CloseHandle(syscall.Handle(handle))
+		}
+	}
+}
+
+// lockPath locks a directory or symlink, return handle, exec "syscall.CloseHandle(handle)" to unlock the path
+func lockPath(path string) (uintptr, error) {
+	if len(path) == 0 {
+		return uintptr(syscall.InvalidHandle), syscall.ERROR_FILE_NOT_FOUND
+	}
+	pathp, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return uintptr(syscall.InvalidHandle), err
+	}
+	access := uint32(syscall.GENERIC_READ)
+	sharemode := uint32(syscall.FILE_SHARE_READ)
+	createmode := uint32(syscall.OPEN_EXISTING)
+	flags := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS | syscall.FILE_FLAG_OPEN_REPARSE_POINT)
+	fd, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, flags, 0)
+	return uintptr(fd), err
+}
+
+// Lock all directories in subPath and check they're not symlinks.
+func (sp *subpath) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
+	handles, err := lockAndCheckSubPath(subPath.VolumePath, subPath.Path)
+
+	// Unlock the directories when the container starts
+	cleanupAction = func() {
+		unlockPath(handles)
+	}
+	return subPath.Path, cleanupAction, err
+}
+
+// No bind-mounts for subpaths are necessary on Windows
+func (sp *subpath) CleanSubPaths(podDir string, volumeName string) error {
+	return nil
+}
+
+// SafeMakeDir makes sure that the created directory does not escape given base directory mis-using symlinks.
+func (sp *subpath) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
+	realBase, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
+	}
+
+	realFullPath := filepath.Join(realBase, subdir)
+	return doSafeMakeDir(realFullPath, realBase, perm)
+}
+
+func doSafeMakeDir(pathname string, base string, perm os.FileMode) error {
+	klog.V(4).Infof("Creating directory %q within base %q", pathname, base)
+
+	if !mount.PathWithinBase(pathname, base) {
+		return fmt.Errorf("path %s is outside of allowed base %s", pathname, base)
+	}
+
+	// Quick check if the directory already exists
+	s, err := os.Stat(pathname)
+	if err == nil {
+		// Path exists
+		if s.IsDir() {
+			// The directory already exists. It can be outside of the parent,
+			// but there is no race-proof check.
+			klog.V(4).Infof("Directory %s already exists", pathname)
+			return nil
+		}
+		return &os.PathError{Op: "mkdir", Path: pathname, Err: syscall.ENOTDIR}
+	}
+
+	// Find all existing directories
+	existingPath, toCreate, err := findExistingPrefix(base, pathname)
+	if err != nil {
+		return fmt.Errorf("error opening directory %s: %s", pathname, err)
+	}
+	if len(toCreate) == 0 {
+		return nil
+	}
+
+	// Ensure the existing directory is inside allowed base
+	fullExistingPath, err := filepath.EvalSymlinks(existingPath)
+	if err != nil {
+		return fmt.Errorf("error opening existing directory %s: %s", existingPath, err)
+	}
+	fullBasePath, err := filepath.EvalSymlinks(base)
+	if err != nil {
+		return fmt.Errorf("cannot read link %s: %s", base, err)
+	}
+	if !mount.PathWithinBase(fullExistingPath, fullBasePath) {
+		return fmt.Errorf("path %s is outside of allowed base %s", fullExistingPath, err)
+	}
+
+	// lock all intermediate directories from fullBasePath to fullExistingPath (top to bottom)
+	fileHandles, err := lockAndCheckSubPathWithoutSymlink(fullBasePath, fullExistingPath)
+	defer unlockPath(fileHandles)
+	if err != nil {
+		return err
+	}
+
+	klog.V(4).Infof("%q already exists, %q to create", fullExistingPath, filepath.Join(toCreate...))
+	currentPath := fullExistingPath
+	// create the directories one by one, making sure nobody can change
+	// created directory into symlink by lock that directory immediately
+	for _, dir := range toCreate {
+		currentPath = filepath.Join(currentPath, dir)
+		klog.V(4).Infof("Creating %s", dir)
+		if err := os.Mkdir(currentPath, perm); err != nil {
+			return fmt.Errorf("cannot create directory %s: %s", currentPath, err)
+		}
+		handle, err := lockPath(currentPath)
+		if err != nil {
+			return fmt.Errorf("cannot lock path %s: %s", currentPath, err)
+		}
+		defer syscall.CloseHandle(syscall.Handle(handle))
+		// make sure newly created directory does not contain symlink after lock
+		stat, err := os.Lstat(currentPath)
+		if err != nil {
+			return fmt.Errorf("Lstat(%q) error: %v", currentPath, err)
+		}
+		if stat.Mode()&os.ModeSymlink != 0 {
+			return fmt.Errorf("subpath %q is an unexpected symlink after Mkdir", currentPath)
+		}
+	}
+
+	return nil
+}
+
+// findExistingPrefix finds prefix of pathname that exists. In addition, it
+// returns list of remaining directories that don't exist yet.
+func findExistingPrefix(base, pathname string) (string, []string, error) {
+	rel, err := filepath.Rel(base, pathname)
+	if err != nil {
+		return base, nil, err
+	}
+
+	if mount.StartsWithBackstep(rel) {
+		return base, nil, fmt.Errorf("pathname(%s) is not within base(%s)", pathname, base)
+	}
+
+	if rel == "." {
+		// base and pathname are equal
+		return pathname, []string{}, nil
+	}
+
+	dirs := strings.Split(rel, string(filepath.Separator))
+
+	parent := base
+	currentPath := base
+	for i, dir := range dirs {
+		parent = currentPath
+		currentPath = filepath.Join(parent, dir)
+		if _, err := os.Lstat(currentPath); err != nil {
+			if os.IsNotExist(err) {
+				return parent, dirs[i:], nil
+			}
+			return base, nil, err
+		}
+	}
+
+	return pathname, []string{}, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/types/types.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package types defines types used only by volume components
+package types
+
+import "k8s.io/apimachinery/pkg/types"
+
+// UniquePodName defines the type to key pods off of
+type UniquePodName types.UID
+
+// UniquePVCName defines the type to key pvc off
+type UniquePVCName types.UID
+
+// GeneratedOperations contains the operation that is created as well as
+// supporting functions required for the operation executor
+type GeneratedOperations struct {
+	OperationFunc     func() (eventErr error, detailedErr error)
+	EventRecorderFunc func(*error)
+	CompleteFunc      func(*error)
+}

vendor/k8s.io/kubernetes/pkg/volume/util/util.go

@@ -0,0 +1,532 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	utypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
+	"k8s.io/kubernetes/pkg/features"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume"
+	"k8s.io/kubernetes/pkg/volume/util/types"
+	"k8s.io/kubernetes/pkg/volume/util/volumepathhandler"
+	utilstrings "k8s.io/utils/strings"
+)
+
+const (
+	readyFileName = "ready"
+
+	// ControllerManagedAttachAnnotation is the key of the annotation on Node
+	// objects that indicates attach/detach operations for the node should be
+	// managed by the attach/detach controller
+	ControllerManagedAttachAnnotation string = "volumes.kubernetes.io/controller-managed-attach-detach"
+
+	// KeepTerminatedPodVolumesAnnotation is the key of the annotation on Node
+	// that decides if pod volumes are unmounted when pod is terminated
+	KeepTerminatedPodVolumesAnnotation string = "volumes.kubernetes.io/keep-terminated-pod-volumes"
+
+	// VolumeGidAnnotationKey is the of the annotation on the PersistentVolume
+	// object that specifies a supplemental GID.
+	VolumeGidAnnotationKey = "pv.beta.kubernetes.io/gid"
+
+	// VolumeDynamicallyCreatedByKey is the key of the annotation on PersistentVolume
+	// object created dynamically
+	VolumeDynamicallyCreatedByKey = "kubernetes.io/createdby"
+)
+
+// IsReady checks for the existence of a regular file
+// called 'ready' in the given directory and returns
+// true if that file exists.
+func IsReady(dir string) bool {
+	readyFile := path.Join(dir, readyFileName)
+	s, err := os.Stat(readyFile)
+	if err != nil {
+		return false
+	}
+
+	if !s.Mode().IsRegular() {
+		klog.Errorf("ready-file is not a file: %s", readyFile)
+		return false
+	}
+
+	return true
+}
+
+// SetReady creates a file called 'ready' in the given
+// directory.  It logs an error if the file cannot be
+// created.
+func SetReady(dir string) {
+	if err := os.MkdirAll(dir, 0750); err != nil && !os.IsExist(err) {
+		klog.Errorf("Can't mkdir %s: %v", dir, err)
+		return
+	}
+
+	readyFile := path.Join(dir, readyFileName)
+	file, err := os.Create(readyFile)
+	if err != nil {
+		klog.Errorf("Can't touch %s: %v", readyFile, err)
+		return
+	}
+	file.Close()
+}
+
+// GetSecretForPod locates secret by name in the pod's namespace and returns secret map
+func GetSecretForPod(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (map[string]string, error) {
+	secret := make(map[string]string)
+	if kubeClient == nil {
+		return secret, fmt.Errorf("Cannot get kube client")
+	}
+	secrets, err := kubeClient.CoreV1().Secrets(pod.Namespace).Get(secretName, metav1.GetOptions{})
+	if err != nil {
+		return secret, err
+	}
+	for name, data := range secrets.Data {
+		secret[name] = string(data)
+	}
+	return secret, nil
+}
+
+// GetSecretForPV locates secret by name and namespace, verifies the secret type, and returns secret map
+func GetSecretForPV(secretNamespace, secretName, volumePluginName string, kubeClient clientset.Interface) (map[string]string, error) {
+	secret := make(map[string]string)
+	if kubeClient == nil {
+		return secret, fmt.Errorf("Cannot get kube client")
+	}
+	secrets, err := kubeClient.CoreV1().Secrets(secretNamespace).Get(secretName, metav1.GetOptions{})
+	if err != nil {
+		return secret, err
+	}
+	if secrets.Type != v1.SecretType(volumePluginName) {
+		return secret, fmt.Errorf("Cannot get secret of type %s", volumePluginName)
+	}
+	for name, data := range secrets.Data {
+		secret[name] = string(data)
+	}
+	return secret, nil
+}
+
+// GetClassForVolume locates storage class by persistent volume
+func GetClassForVolume(kubeClient clientset.Interface, pv *v1.PersistentVolume) (*storage.StorageClass, error) {
+	if kubeClient == nil {
+		return nil, fmt.Errorf("Cannot get kube client")
+	}
+	className := v1helper.GetPersistentVolumeClass(pv)
+	if className == "" {
+		return nil, fmt.Errorf("Volume has no storage class")
+	}
+
+	class, err := kubeClient.StorageV1().StorageClasses().Get(className, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+	return class, nil
+}
+
+// CheckNodeAffinity looks at the PV node affinity, and checks if the node has the same corresponding labels
+// This ensures that we don't mount a volume that doesn't belong to this node
+func CheckNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
+	return checkVolumeNodeAffinity(pv, nodeLabels)
+}
+
+func checkVolumeNodeAffinity(pv *v1.PersistentVolume, nodeLabels map[string]string) error {
+	if pv.Spec.NodeAffinity == nil {
+		return nil
+	}
+
+	if pv.Spec.NodeAffinity.Required != nil {
+		terms := pv.Spec.NodeAffinity.Required.NodeSelectorTerms
+		klog.V(10).Infof("Match for Required node selector terms %+v", terms)
+		if !v1helper.MatchNodeSelectorTerms(terms, labels.Set(nodeLabels), nil) {
+			return fmt.Errorf("No matching NodeSelectorTerms")
+		}
+	}
+
+	return nil
+}
+
+// LoadPodFromFile will read, decode, and return a Pod from a file.
+func LoadPodFromFile(filePath string) (*v1.Pod, error) {
+	if filePath == "" {
+		return nil, fmt.Errorf("file path not specified")
+	}
+	podDef, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file path %s: %+v", filePath, err)
+	}
+	if len(podDef) == 0 {
+		return nil, fmt.Errorf("file was empty: %s", filePath)
+	}
+	pod := &v1.Pod{}
+
+	codec := legacyscheme.Codecs.UniversalDecoder()
+	if err := runtime.DecodeInto(codec, podDef, pod); err != nil {
+		return nil, fmt.Errorf("failed decoding file: %v", err)
+	}
+	return pod, nil
+}
+
+// CalculateTimeoutForVolume calculates time for a Recycler pod to complete a
+// recycle operation. The calculation and return value is either the
+// minimumTimeout or the timeoutIncrement per Gi of storage size, whichever is
+// greater.
+func CalculateTimeoutForVolume(minimumTimeout, timeoutIncrement int, pv *v1.PersistentVolume) int64 {
+	giQty := resource.MustParse("1Gi")
+	pvQty := pv.Spec.Capacity[v1.ResourceStorage]
+	giSize := giQty.Value()
+	pvSize := pvQty.Value()
+	timeout := (pvSize / giSize) * int64(timeoutIncrement)
+	if timeout < int64(minimumTimeout) {
+		return int64(minimumTimeout)
+	}
+	return timeout
+}
+
+// GenerateVolumeName returns a PV name with clusterName prefix. The function
+// should be used to generate a name of GCE PD or Cinder volume. It basically
+// adds "<clusterName>-dynamic-" before the PV name, making sure the resulting
+// string fits given length and cuts "dynamic" if not.
+func GenerateVolumeName(clusterName, pvName string, maxLength int) string {
+	prefix := clusterName + "-dynamic"
+	pvLen := len(pvName)
+
+	// cut the "<clusterName>-dynamic" to fit full pvName into maxLength
+	// +1 for the '-' dash
+	if pvLen+1+len(prefix) > maxLength {
+		prefix = prefix[:maxLength-pvLen-1]
+	}
+	return prefix + "-" + pvName
+}
+
+// GetPath checks if the path from the mounter is empty.
+func GetPath(mounter volume.Mounter) (string, error) {
+	path := mounter.GetPath()
+	if path == "" {
+		return "", fmt.Errorf("Path is empty %s", reflect.TypeOf(mounter).String())
+	}
+	return path, nil
+}
+
+// UnmountViaEmptyDir delegates the tear down operation for secret, configmap, git_repo and downwardapi
+// to empty_dir
+func UnmountViaEmptyDir(dir string, host volume.VolumeHost, volName string, volSpec volume.Spec, podUID utypes.UID) error {
+	klog.V(3).Infof("Tearing down volume %v for pod %v at %v", volName, podUID, dir)
+
+	// Wrap EmptyDir, let it do the teardown.
+	wrapped, err := host.NewWrapperUnmounter(volName, volSpec, podUID)
+	if err != nil {
+		return err
+	}
+	return wrapped.TearDownAt(dir)
+}
+
+// MountOptionFromSpec extracts and joins mount options from volume spec with supplied options
+func MountOptionFromSpec(spec *volume.Spec, options ...string) []string {
+	pv := spec.PersistentVolume
+
+	if pv != nil {
+		// Use beta annotation first
+		if mo, ok := pv.Annotations[v1.MountOptionAnnotation]; ok {
+			moList := strings.Split(mo, ",")
+			return JoinMountOptions(moList, options)
+		}
+
+		if len(pv.Spec.MountOptions) > 0 {
+			return JoinMountOptions(pv.Spec.MountOptions, options)
+		}
+	}
+
+	return options
+}
+
+// JoinMountOptions joins mount options eliminating duplicates
+func JoinMountOptions(userOptions []string, systemOptions []string) []string {
+	allMountOptions := sets.NewString()
+
+	for _, mountOption := range userOptions {
+		if len(mountOption) > 0 {
+			allMountOptions.Insert(mountOption)
+		}
+	}
+
+	for _, mountOption := range systemOptions {
+		allMountOptions.Insert(mountOption)
+	}
+	return allMountOptions.List()
+}
+
+// AccessModesContains returns whether the requested mode is contained by modes
+func AccessModesContains(modes []v1.PersistentVolumeAccessMode, mode v1.PersistentVolumeAccessMode) bool {
+	for _, m := range modes {
+		if m == mode {
+			return true
+		}
+	}
+	return false
+}
+
+// AccessModesContainedInAll returns whether all of the requested modes are contained by modes
+func AccessModesContainedInAll(indexedModes []v1.PersistentVolumeAccessMode, requestedModes []v1.PersistentVolumeAccessMode) bool {
+	for _, mode := range requestedModes {
+		if !AccessModesContains(indexedModes, mode) {
+			return false
+		}
+	}
+	return true
+}
+
+// GetWindowsPath get a windows path
+func GetWindowsPath(path string) string {
+	windowsPath := strings.Replace(path, "/", "\\", -1)
+	if strings.HasPrefix(windowsPath, "\\") {
+		windowsPath = "c:" + windowsPath
+	}
+	return windowsPath
+}
+
+// GetUniquePodName returns a unique identifier to reference a pod by
+func GetUniquePodName(pod *v1.Pod) types.UniquePodName {
+	return types.UniquePodName(pod.UID)
+}
+
+// GetUniqueVolumeName returns a unique name representing the volume/plugin.
+// Caller should ensure that volumeName is a name/ID uniquely identifying the
+// actual backing device, directory, path, etc. for a particular volume.
+// The returned name can be used to uniquely reference the volume, for example,
+// to prevent operations (attach/detach or mount/unmount) from being triggered
+// on the same volume.
+func GetUniqueVolumeName(pluginName, volumeName string) v1.UniqueVolumeName {
+	return v1.UniqueVolumeName(fmt.Sprintf("%s/%s", pluginName, volumeName))
+}
+
+// GetUniqueVolumeNameFromSpecWithPod returns a unique volume name with pod
+// name included. This is useful to generate different names for different pods
+// on same volume.
+func GetUniqueVolumeNameFromSpecWithPod(
+	podName types.UniquePodName, volumePlugin volume.VolumePlugin, volumeSpec *volume.Spec) v1.UniqueVolumeName {
+	return v1.UniqueVolumeName(
+		fmt.Sprintf("%s/%v-%s", volumePlugin.GetPluginName(), podName, volumeSpec.Name()))
+}
+
+// GetUniqueVolumeNameFromSpec uses the given VolumePlugin to generate a unique
+// name representing the volume defined in the specified volume spec.
+// This returned name can be used to uniquely reference the actual backing
+// device, directory, path, etc. referenced by the given volumeSpec.
+// If the given plugin does not support the volume spec, this returns an error.
+func GetUniqueVolumeNameFromSpec(
+	volumePlugin volume.VolumePlugin,
+	volumeSpec *volume.Spec) (v1.UniqueVolumeName, error) {
+	if volumePlugin == nil {
+		return "", fmt.Errorf(
+			"volumePlugin should not be nil. volumeSpec.Name=%q",
+			volumeSpec.Name())
+	}
+
+	volumeName, err := volumePlugin.GetVolumeName(volumeSpec)
+	if err != nil || volumeName == "" {
+		return "", fmt.Errorf(
+			"failed to GetVolumeName from volumePlugin for volumeSpec %q err=%v",
+			volumeSpec.Name(),
+			err)
+	}
+
+	return GetUniqueVolumeName(
+			volumePlugin.GetPluginName(),
+			volumeName),
+		nil
+}
+
+// IsPodTerminated checks if pod is terminated
+func IsPodTerminated(pod *v1.Pod, podStatus v1.PodStatus) bool {
+	return podStatus.Phase == v1.PodFailed || podStatus.Phase == v1.PodSucceeded || (pod.DeletionTimestamp != nil && notRunning(podStatus.ContainerStatuses))
+}
+
+// notRunning returns true if every status is terminated or waiting, or the status list
+// is empty.
+func notRunning(statuses []v1.ContainerStatus) bool {
+	for _, status := range statuses {
+		if status.State.Terminated == nil && status.State.Waiting == nil {
+			return false
+		}
+	}
+	return true
+}
+
+// SplitUniqueName splits the unique name to plugin name and volume name strings. It expects the uniqueName to follow
+// the format plugin_name/volume_name and the plugin name must be namespaced as described by the plugin interface,
+// i.e. namespace/plugin containing exactly one '/'. This means the unique name will always be in the form of
+// plugin_namespace/plugin/volume_name, see k8s.io/kubernetes/pkg/volume/plugins.go VolumePlugin interface
+// description and pkg/volume/util/volumehelper/volumehelper.go GetUniqueVolumeNameFromSpec that constructs
+// the unique volume names.
+func SplitUniqueName(uniqueName v1.UniqueVolumeName) (string, string, error) {
+	components := strings.SplitN(string(uniqueName), "/", 3)
+	if len(components) != 3 {
+		return "", "", fmt.Errorf("cannot split volume unique name %s to plugin/volume components", uniqueName)
+	}
+	pluginName := fmt.Sprintf("%s/%s", components[0], components[1])
+	return pluginName, components[2], nil
+}
+
+// NewSafeFormatAndMountFromHost creates a new SafeFormatAndMount with Mounter
+// and Exec taken from given VolumeHost.
+func NewSafeFormatAndMountFromHost(pluginName string, host volume.VolumeHost) *mount.SafeFormatAndMount {
+	mounter := host.GetMounter(pluginName)
+	exec := host.GetExec(pluginName)
+	return &mount.SafeFormatAndMount{Interface: mounter, Exec: exec}
+}
+
+// GetVolumeMode retrieves VolumeMode from pv.
+// If the volume doesn't have PersistentVolume, it's an inline volume,
+// should return volumeMode as filesystem to keep existing behavior.
+func GetVolumeMode(volumeSpec *volume.Spec) (v1.PersistentVolumeMode, error) {
+	if volumeSpec == nil || volumeSpec.PersistentVolume == nil {
+		return v1.PersistentVolumeFilesystem, nil
+	}
+	if volumeSpec.PersistentVolume.Spec.VolumeMode != nil {
+		return *volumeSpec.PersistentVolume.Spec.VolumeMode, nil
+	}
+	return "", fmt.Errorf("cannot get volumeMode for volume: %v", volumeSpec.Name())
+}
+
+// GetPersistentVolumeClaimVolumeMode retrieves VolumeMode from pvc.
+func GetPersistentVolumeClaimVolumeMode(claim *v1.PersistentVolumeClaim) (v1.PersistentVolumeMode, error) {
+	if claim.Spec.VolumeMode != nil {
+		return *claim.Spec.VolumeMode, nil
+	}
+	return "", fmt.Errorf("cannot get volumeMode from pvc: %v", claim.Name)
+}
+
+// GetPersistentVolumeClaimQualifiedName returns a qualified name for pvc.
+func GetPersistentVolumeClaimQualifiedName(claim *v1.PersistentVolumeClaim) string {
+	return utilstrings.JoinQualifiedName(claim.GetNamespace(), claim.GetName())
+}
+
+// CheckVolumeModeFilesystem checks VolumeMode.
+// If the mode is Filesystem, return true otherwise return false.
+func CheckVolumeModeFilesystem(volumeSpec *volume.Spec) (bool, error) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) {
+		volumeMode, err := GetVolumeMode(volumeSpec)
+		if err != nil {
+			return true, err
+		}
+		if volumeMode == v1.PersistentVolumeBlock {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+// CheckPersistentVolumeClaimModeBlock checks VolumeMode.
+// If the mode is Block, return true otherwise return false.
+func CheckPersistentVolumeClaimModeBlock(pvc *v1.PersistentVolumeClaim) bool {
+	return utilfeature.DefaultFeatureGate.Enabled(features.BlockVolume) && pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == v1.PersistentVolumeBlock
+}
+
+// IsWindowsUNCPath checks if path is prefixed with \\
+// This can be used to skip any processing of paths
+// that point to SMB shares, local named pipes and local UNC path
+func IsWindowsUNCPath(goos, path string) bool {
+	if goos != "windows" {
+		return false
+	}
+	// Check for UNC prefix \\
+	if strings.HasPrefix(path, `\\`) {
+		return true
+	}
+	return false
+}
+
+// IsWindowsLocalPath checks if path is a local path
+// prefixed with "/" or "\" like "/foo/bar" or "\foo\bar"
+func IsWindowsLocalPath(goos, path string) bool {
+	if goos != "windows" {
+		return false
+	}
+	if IsWindowsUNCPath(goos, path) {
+		return false
+	}
+	if strings.Contains(path, ":") {
+		return false
+	}
+	if !(strings.HasPrefix(path, `/`) || strings.HasPrefix(path, `\`)) {
+		return false
+	}
+	return true
+}
+
+// MakeAbsolutePath convert path to absolute path according to GOOS
+func MakeAbsolutePath(goos, path string) string {
+	if goos != "windows" {
+		return filepath.Clean("/" + path)
+	}
+	// These are all for windows
+	// If there is a colon, give up.
+	if strings.Contains(path, ":") {
+		return path
+	}
+	// If there is a slash, but no drive, add 'c:'
+	if strings.HasPrefix(path, "/") || strings.HasPrefix(path, "\\") {
+		return "c:" + path
+	}
+	// Otherwise, add 'c:\'
+	return "c:\\" + path
+}
+
+// MapBlockVolume is a utility function to provide a common way of mounting
+// block device path for a specified volume and pod.  This function should be
+// called by volume plugins that implements volume.BlockVolumeMapper.Map() method.
+func MapBlockVolume(
+	devicePath,
+	globalMapPath,
+	podVolumeMapPath,
+	volumeMapName string,
+	podUID utypes.UID,
+) error {
+	blkUtil := volumepathhandler.NewBlockVolumePathHandler()
+
+	// map devicePath to global node path
+	mapErr := blkUtil.MapDevice(devicePath, globalMapPath, string(podUID))
+	if mapErr != nil {
+		return mapErr
+	}
+
+	// map devicePath to pod volume path
+	mapErr = blkUtil.MapDevice(devicePath, podVolumeMapPath, volumeMapName)
+	if mapErr != nil {
+		return mapErr
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volumepathhandler
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	"k8s.io/klog"
+
+	"k8s.io/apimachinery/pkg/types"
+)
+
+const (
+	losetupPath           = "losetup"
+	ErrDeviceNotFound     = "device not found"
+	ErrDeviceNotSupported = "device not supported"
+)
+
+// BlockVolumePathHandler defines a set of operations for handling block volume-related operations
+type BlockVolumePathHandler interface {
+	// MapDevice creates a symbolic link to block device under specified map path
+	MapDevice(devicePath string, mapPath string, linkName string) error
+	// UnmapDevice removes a symbolic link to block device under specified map path
+	UnmapDevice(mapPath string, linkName string) error
+	// RemovePath removes a file or directory on specified map path
+	RemoveMapPath(mapPath string) error
+	// IsSymlinkExist retruns true if specified symbolic link exists
+	IsSymlinkExist(mapPath string) (bool, error)
+	// GetDeviceSymlinkRefs searches symbolic links under global map path
+	GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error)
+	// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
+	// corresponding to map path symlink, and then return global map path with pod uuid.
+	FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error)
+	// AttachFileDevice takes a path to a regular file and makes it available as an
+	// attached block device.
+	AttachFileDevice(path string) (string, error)
+	// GetLoopDevice returns the full path to the loop device associated with the given path.
+	GetLoopDevice(path string) (string, error)
+	// RemoveLoopDevice removes specified loopback device
+	RemoveLoopDevice(device string) error
+}
+
+// NewBlockVolumePathHandler returns a new instance of BlockVolumeHandler.
+func NewBlockVolumePathHandler() BlockVolumePathHandler {
+	var volumePathHandler VolumePathHandler
+	return volumePathHandler
+}
+
+// VolumePathHandler is path related operation handlers for block volume
+type VolumePathHandler struct {
+}
+
+// MapDevice creates a symbolic link to block device under specified map path
+func (v VolumePathHandler) MapDevice(devicePath string, mapPath string, linkName string) error {
+	// Example of global map path:
+	//   globalMapPath/linkName: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{podUid}
+	//   linkName: {podUid}
+	//
+	// Example of pod device map path:
+	//   podDeviceMapPath/linkName: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName}
+	//   linkName: {volumeName}
+	if len(devicePath) == 0 {
+		return fmt.Errorf("Failed to map device to map path. devicePath is empty")
+	}
+	if len(mapPath) == 0 {
+		return fmt.Errorf("Failed to map device to map path. mapPath is empty")
+	}
+	if !filepath.IsAbs(mapPath) {
+		return fmt.Errorf("The map path should be absolute: map path: %s", mapPath)
+	}
+	klog.V(5).Infof("MapDevice: devicePath %s", devicePath)
+	klog.V(5).Infof("MapDevice: mapPath %s", mapPath)
+	klog.V(5).Infof("MapDevice: linkName %s", linkName)
+
+	// Check and create mapPath
+	_, err := os.Stat(mapPath)
+	if err != nil && !os.IsNotExist(err) {
+		klog.Errorf("cannot validate map path: %s", mapPath)
+		return err
+	}
+	if err = os.MkdirAll(mapPath, 0750); err != nil {
+		return fmt.Errorf("Failed to mkdir %s, error %v", mapPath, err)
+	}
+	// Remove old symbolic link(or file) then create new one.
+	// This should be done because current symbolic link is
+	// stale across node reboot.
+	linkPath := path.Join(mapPath, string(linkName))
+	if err = os.Remove(linkPath); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	err = os.Symlink(devicePath, linkPath)
+	return err
+}
+
+// UnmapDevice removes a symbolic link associated to block device under specified map path
+func (v VolumePathHandler) UnmapDevice(mapPath string, linkName string) error {
+	if len(mapPath) == 0 {
+		return fmt.Errorf("Failed to unmap device from map path. mapPath is empty")
+	}
+	klog.V(5).Infof("UnmapDevice: mapPath %s", mapPath)
+	klog.V(5).Infof("UnmapDevice: linkName %s", linkName)
+
+	// Check symbolic link exists
+	linkPath := path.Join(mapPath, string(linkName))
+	if islinkExist, checkErr := v.IsSymlinkExist(linkPath); checkErr != nil {
+		return checkErr
+	} else if !islinkExist {
+		klog.Warningf("Warning: Unmap skipped because symlink does not exist on the path: %v", linkPath)
+		return nil
+	}
+	err := os.Remove(linkPath)
+	return err
+}
+
+// RemoveMapPath removes a file or directory on specified map path
+func (v VolumePathHandler) RemoveMapPath(mapPath string) error {
+	if len(mapPath) == 0 {
+		return fmt.Errorf("Failed to remove map path. mapPath is empty")
+	}
+	klog.V(5).Infof("RemoveMapPath: mapPath %s", mapPath)
+	err := os.RemoveAll(mapPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	}
+	return nil
+}
+
+// IsSymlinkExist returns true if specified file exists and the type is symbolik link.
+// If file doesn't exist, or file exists but not symbolic link, return false with no error.
+// On other cases, return false with error from Lstat().
+func (v VolumePathHandler) IsSymlinkExist(mapPath string) (bool, error) {
+	fi, err := os.Lstat(mapPath)
+	if err == nil {
+		// If file exits and it's symbolic link, return true and no error
+		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			return true, nil
+		}
+		// If file exits but it's not symbolic link, return fale and no error
+		return false, nil
+	}
+	// If file doesn't exist, return false and no error
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	// Return error from Lstat()
+	return false, err
+}
+
+// GetDeviceSymlinkRefs searches symbolic links under global map path
+func (v VolumePathHandler) GetDeviceSymlinkRefs(devPath string, mapPath string) ([]string, error) {
+	var refs []string
+	files, err := ioutil.ReadDir(mapPath)
+	if err != nil {
+		return nil, fmt.Errorf("Directory cannot read %v", err)
+	}
+	for _, file := range files {
+		if file.Mode()&os.ModeSymlink != os.ModeSymlink {
+			continue
+		}
+		filename := file.Name()
+		filepath, err := os.Readlink(path.Join(mapPath, filename))
+		if err != nil {
+			return nil, fmt.Errorf("Symbolic link cannot be retrieved %v", err)
+		}
+		klog.V(5).Infof("GetDeviceSymlinkRefs: filepath: %v, devPath: %v", filepath, devPath)
+		if filepath == devPath {
+			refs = append(refs, path.Join(mapPath, filename))
+		}
+	}
+	klog.V(5).Infof("GetDeviceSymlinkRefs: refs %v", refs)
+	return refs, nil
+}
+
+// FindGlobalMapPathUUIDFromPod finds {pod uuid} symbolic link under globalMapPath
+// corresponding to map path symlink, and then return global map path with pod uuid.
+// ex. mapPath symlink: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} -> /dev/sdX
+//     globalMapPath/{pod uuid}: plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} -> /dev/sdX
+func (v VolumePathHandler) FindGlobalMapPathUUIDFromPod(pluginDir, mapPath string, podUID types.UID) (string, error) {
+	var globalMapPathUUID string
+	// Find symbolic link named pod uuid under plugin dir
+	err := filepath.Walk(pluginDir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if (fi.Mode()&os.ModeSymlink == os.ModeSymlink) && (fi.Name() == string(podUID)) {
+			klog.V(5).Infof("FindGlobalMapPathFromPod: path %s, mapPath %s", path, mapPath)
+			if res, err := compareSymlinks(path, mapPath); err == nil && res {
+				globalMapPathUUID = path
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+	klog.V(5).Infof("FindGlobalMapPathFromPod: globalMapPathUUID %s", globalMapPathUUID)
+	// Return path contains global map path + {pod uuid}
+	return globalMapPathUUID, nil
+}
+
+func compareSymlinks(global, pod string) (bool, error) {
+	devGlobal, err := os.Readlink(global)
+	if err != nil {
+		return false, err
+	}
+	devPod, err := os.Readlink(pod)
+	if err != nil {
+		return false, err
+	}
+	klog.V(5).Infof("CompareSymlinks: devGloBal %s, devPod %s", devGlobal, devPod)
+	if devGlobal == devPod {
+		return true, nil
+	}
+	return false, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler_linux.go

@@ -0,0 +1,108 @@
+// +build linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volumepathhandler
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"k8s.io/klog"
+)
+
+// AttachFileDevice takes a path to a regular file and makes it available as an
+// attached block device.
+func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
+	blockDevicePath, err := v.GetLoopDevice(path)
+	if err != nil && err.Error() != ErrDeviceNotFound {
+		return "", err
+	}
+
+	// If no existing loop device for the path, create one
+	if blockDevicePath == "" {
+		klog.V(4).Infof("Creating device for path: %s", path)
+		blockDevicePath, err = makeLoopDevice(path)
+		if err != nil {
+			return "", err
+		}
+	}
+	return blockDevicePath, nil
+}
+
+// GetLoopDevice returns the full path to the loop device associated with the given path.
+func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
+	_, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		return "", errors.New(ErrDeviceNotFound)
+	}
+	if err != nil {
+		return "", fmt.Errorf("not attachable: %v", err)
+	}
+
+	args := []string{"-j", path}
+	cmd := exec.Command(losetupPath, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		klog.V(2).Infof("Failed device discover command for path %s: %v %s", path, err, out)
+		return "", err
+	}
+	return parseLosetupOutputForDevice(out)
+}
+
+func makeLoopDevice(path string) (string, error) {
+	args := []string{"-f", "--show", path}
+	cmd := exec.Command(losetupPath, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		klog.V(2).Infof("Failed device create command for path: %s %v %s ", path, err, out)
+		return "", err
+	}
+	return parseLosetupOutputForDevice(out)
+}
+
+// RemoveLoopDevice removes specified loopback device
+func (v VolumePathHandler) RemoveLoopDevice(device string) error {
+	args := []string{"-d", device}
+	cmd := exec.Command(losetupPath, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		if _, err := os.Stat(device); os.IsNotExist(err) {
+			return nil
+		}
+		klog.V(2).Infof("Failed to remove loopback device: %s: %v %s", device, err, out)
+		return err
+	}
+	return nil
+}
+
+func parseLosetupOutputForDevice(output []byte) (string, error) {
+	if len(output) == 0 {
+		return "", errors.New(ErrDeviceNotFound)
+	}
+
+	// losetup returns device in the format:
+	// /dev/loop1: [0073]:148662 (/dev/sda)
+	device := strings.TrimSpace(strings.SplitN(string(output), ":", 2)[0])
+	if len(device) == 0 {
+		return "", errors.New(ErrDeviceNotFound)
+	}
+	return device, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/util/volumepathhandler/volume_path_handler_unsupported.go

@@ -0,0 +1,39 @@
+// +build !linux
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package volumepathhandler
+
+import (
+	"fmt"
+)
+
+// AttachFileDevice takes a path to a regular file and makes it available as an
+// attached block device.
+func (v VolumePathHandler) AttachFileDevice(path string) (string, error) {
+	return "", fmt.Errorf("AttachFileDevice not supported for this build.")
+}
+
+// GetLoopDevice returns the full path to the loop device associated with the given path.
+func (v VolumePathHandler) GetLoopDevice(path string) (string, error) {
+	return "", fmt.Errorf("GetLoopDevice not supported for this build.")
+}
+
+// RemoveLoopDevice removes specified loopback device
+func (v VolumePathHandler) RemoveLoopDevice(device string) error {
+	return fmt.Errorf("RemoveLoopDevice not supported for this build.")
+}