This commit adds "k8s.io/kubernetes/test/e2e/framework/config"

and its functions in E2E.

update vendor packages
log dismounter command output
use kube v1.17.1 in dependency

Signed-off-by: Humble Chirammal <hchiramm@redhat.com>

vendor/k8s.io/apiserver/pkg/apis/apiserver/install/install.go

@@ -0,0 +1,38 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package install
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/v1"
+	"k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+)
+
+// Install registers the API group and adds types to a scheme
+func Install(scheme *runtime.Scheme) {
+	utilruntime.Must(apiserver.AddToScheme(scheme))
+
+	// v1alpha is in the k8s.io-suffixed API group
+	utilruntime.Must(v1alpha1.AddToScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1alpha1.SchemeGroupVersion))
+
+	// v1 is in the config.k8s.io-suffixed API group
+	utilruntime.Must(v1.AddToScheme(scheme))
+	utilruntime.Must(scheme.SetVersionPriority(v1.SchemeGroupVersion))
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/doc.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+// +k8s:conversion-gen=k8s.io/apiserver/pkg/apis/apiserver
+// +k8s:defaulter-gen=TypeMeta
+// +groupName=apiserver.k8s.io
+
+// Package v1alpha1 is the v1alpha1 version of the API.
+package v1alpha1 // import "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+const GroupName = "apiserver.k8s.io"
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
+
+var (
+	// TODO: move SchemeBuilder with zz_generated.deepcopy.go to k8s.io/api.
+	// localSchemeBuilder and AddToScheme will stay in k8s.io/kubernetes.
+	SchemeBuilder      runtime.SchemeBuilder
+	localSchemeBuilder = &SchemeBuilder
+	AddToScheme        = localSchemeBuilder.AddToScheme
+)
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	localSchemeBuilder.Register(addKnownTypes)
+}
+
+// Adds the list of known types to the given scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&AdmissionConfiguration{},
+		&EgressSelectorConfiguration{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go

@@ -0,0 +1,110 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// AdmissionConfiguration provides versioned configuration for admission controllers.
+type AdmissionConfiguration struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// Plugins allows specifying a configuration per admission control plugin.
+	// +optional
+	Plugins []AdmissionPluginConfiguration `json:"plugins"`
+}
+
+// AdmissionPluginConfiguration provides the configuration for a single plug-in.
+type AdmissionPluginConfiguration struct {
+	// Name is the name of the admission controller.
+	// It must match the registered admission plugin name.
+	Name string `json:"name"`
+
+	// Path is the path to a configuration file that contains the plugin's
+	// configuration
+	// +optional
+	Path string `json:"path"`
+
+	// Configuration is an embedded configuration object to be used as the plugin's
+	// configuration. If present, it will be used instead of the path to the configuration file.
+	// +optional
+	Configuration *runtime.Unknown `json:"configuration"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// EgressSelectorConfiguration provides versioned configuration for egress selector clients.
+type EgressSelectorConfiguration struct {
+	metav1.TypeMeta `json:",inline"`
+
+	// connectionServices contains a list of egress selection client configurations
+	EgressSelections []EgressSelection `json:"egressSelections"`
+}
+
+// EgressSelection provides the configuration for a single egress selection client.
+type EgressSelection struct {
+	// name is the name of the egress selection.
+	// Currently supported values are "Master", "Etcd" and "Cluster"
+	Name string `json:"name"`
+
+	// connection is the exact information used to configure the egress selection
+	Connection Connection `json:"connection"`
+}
+
+// Connection provides the configuration for a single egress selection client.
+type Connection struct {
+	// type is the type of connection used to connect from client to network/konnectivity server.
+	// Currently supported values are "http-connect" and "direct".
+	Type string `json:"type"`
+
+	// httpConnect is the config needed to use http-connect to the konnectivity server.
+	// Absence when the type is "http-connect" will cause an error
+	// Presence when the type is "direct" will also cause an error
+	// +optional
+	HTTPConnect *HTTPConnectConfig `json:"httpConnect,omitempty"`
+}
+
+type HTTPConnectConfig struct {
+	// url is the location of the proxy server to connect to.
+	// As an example it might be "https://127.0.0.1:8131"
+	URL string `json:"url"`
+
+	// caBundle is the file location of the CA to be used to determine trust with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	CABundle string `json:"caBundle,omitempty"`
+
+	// clientKey is the file location of the client key to be used in mtls handshakes with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	ClientKey string `json:"clientKey,omitempty"`
+
+	// clientCert is the file location of the client certificate to be used in mtls handshakes with the konnectivity server.
+	// Must be absent/empty http-connect using the plain http
+	// Must be configured for http-connect using the https protocol
+	// Misconfiguration will cause an error
+	// +optional
+	ClientCert string `json:"clientCert,omitempty"`
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go

@@ -0,0 +1,237 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by conversion-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	unsafe "unsafe"
+
+	conversion "k8s.io/apimachinery/pkg/conversion"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	apiserver "k8s.io/apiserver/pkg/apis/apiserver"
+)
+
+func init() {
+	localSchemeBuilder.Register(RegisterConversions)
+}
+
+// RegisterConversions adds conversion functions to the given scheme.
+// Public to allow building arbitrary schemes.
+func RegisterConversions(s *runtime.Scheme) error {
+	if err := s.AddGeneratedConversionFunc((*AdmissionConfiguration)(nil), (*apiserver.AdmissionConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_AdmissionConfiguration_To_apiserver_AdmissionConfiguration(a.(*AdmissionConfiguration), b.(*apiserver.AdmissionConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.AdmissionConfiguration)(nil), (*AdmissionConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfiguration(a.(*apiserver.AdmissionConfiguration), b.(*AdmissionConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*AdmissionPluginConfiguration)(nil), (*apiserver.AdmissionPluginConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_AdmissionPluginConfiguration_To_apiserver_AdmissionPluginConfiguration(a.(*AdmissionPluginConfiguration), b.(*apiserver.AdmissionPluginConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.AdmissionPluginConfiguration)(nil), (*AdmissionPluginConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(a.(*apiserver.AdmissionPluginConfiguration), b.(*AdmissionPluginConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*Connection)(nil), (*apiserver.Connection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_Connection_To_apiserver_Connection(a.(*Connection), b.(*apiserver.Connection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.Connection)(nil), (*Connection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_Connection_To_v1alpha1_Connection(a.(*apiserver.Connection), b.(*Connection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*EgressSelection)(nil), (*apiserver.EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(a.(*EgressSelection), b.(*apiserver.EgressSelection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelection)(nil), (*EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(a.(*apiserver.EgressSelection), b.(*EgressSelection), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*EgressSelectorConfiguration)(nil), (*apiserver.EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(a.(*EgressSelectorConfiguration), b.(*apiserver.EgressSelectorConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.EgressSelectorConfiguration)(nil), (*EgressSelectorConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(a.(*apiserver.EgressSelectorConfiguration), b.(*EgressSelectorConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*HTTPConnectConfig)(nil), (*apiserver.HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(a.(*HTTPConnectConfig), b.(*apiserver.HTTPConnectConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.HTTPConnectConfig)(nil), (*HTTPConnectConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(a.(*apiserver.HTTPConnectConfig), b.(*HTTPConnectConfig), scope)
+	}); err != nil {
+		return err
+	}
+	return nil
+}
+
+func autoConvert_v1alpha1_AdmissionConfiguration_To_apiserver_AdmissionConfiguration(in *AdmissionConfiguration, out *apiserver.AdmissionConfiguration, s conversion.Scope) error {
+	out.Plugins = *(*[]apiserver.AdmissionPluginConfiguration)(unsafe.Pointer(&in.Plugins))
+	return nil
+}
+
+// Convert_v1alpha1_AdmissionConfiguration_To_apiserver_AdmissionConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_AdmissionConfiguration_To_apiserver_AdmissionConfiguration(in *AdmissionConfiguration, out *apiserver.AdmissionConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AdmissionConfiguration_To_apiserver_AdmissionConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfiguration(in *apiserver.AdmissionConfiguration, out *AdmissionConfiguration, s conversion.Scope) error {
+	out.Plugins = *(*[]AdmissionPluginConfiguration)(unsafe.Pointer(&in.Plugins))
+	return nil
+}
+
+// Convert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfiguration is an autogenerated conversion function.
+func Convert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfiguration(in *apiserver.AdmissionConfiguration, out *AdmissionConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_AdmissionConfiguration_To_v1alpha1_AdmissionConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_AdmissionPluginConfiguration_To_apiserver_AdmissionPluginConfiguration(in *AdmissionPluginConfiguration, out *apiserver.AdmissionPluginConfiguration, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Path = in.Path
+	out.Configuration = (*runtime.Unknown)(unsafe.Pointer(in.Configuration))
+	return nil
+}
+
+// Convert_v1alpha1_AdmissionPluginConfiguration_To_apiserver_AdmissionPluginConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_AdmissionPluginConfiguration_To_apiserver_AdmissionPluginConfiguration(in *AdmissionPluginConfiguration, out *apiserver.AdmissionPluginConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AdmissionPluginConfiguration_To_apiserver_AdmissionPluginConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in *apiserver.AdmissionPluginConfiguration, out *AdmissionPluginConfiguration, s conversion.Scope) error {
+	out.Name = in.Name
+	out.Path = in.Path
+	out.Configuration = (*runtime.Unknown)(unsafe.Pointer(in.Configuration))
+	return nil
+}
+
+// Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration is an autogenerated conversion function.
+func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in *apiserver.AdmissionPluginConfiguration, out *AdmissionPluginConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error {
+	out.Type = in.Type
+	out.HTTPConnect = (*apiserver.HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect))
+	return nil
+}
+
+// Convert_v1alpha1_Connection_To_apiserver_Connection is an autogenerated conversion function.
+func Convert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error {
+	return autoConvert_v1alpha1_Connection_To_apiserver_Connection(in, out, s)
+}
+
+func autoConvert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error {
+	out.Type = in.Type
+	out.HTTPConnect = (*HTTPConnectConfig)(unsafe.Pointer(in.HTTPConnect))
+	return nil
+}
+
+// Convert_apiserver_Connection_To_v1alpha1_Connection is an autogenerated conversion function.
+func Convert_apiserver_Connection_To_v1alpha1_Connection(in *apiserver.Connection, out *Connection, s conversion.Scope) error {
+	return autoConvert_apiserver_Connection_To_v1alpha1_Connection(in, out, s)
+}
+
+func autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error {
+	out.Name = in.Name
+	if err := Convert_v1alpha1_Connection_To_apiserver_Connection(&in.Connection, &out.Connection, s); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection is an autogenerated conversion function.
+func Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in *EgressSelection, out *apiserver.EgressSelection, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(in, out, s)
+}
+
+func autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error {
+	out.Name = in.Name
+	if err := Convert_apiserver_Connection_To_v1alpha1_Connection(&in.Connection, &out.Connection, s); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection is an autogenerated conversion function.
+func Convert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in *apiserver.EgressSelection, out *EgressSelection, s conversion.Scope) error {
+	return autoConvert_apiserver_EgressSelection_To_v1alpha1_EgressSelection(in, out, s)
+}
+
+func autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error {
+	out.EgressSelections = *(*[]apiserver.EgressSelection)(unsafe.Pointer(&in.EgressSelections))
+	return nil
+}
+
+// Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in *EgressSelectorConfiguration, out *apiserver.EgressSelectorConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_EgressSelectorConfiguration_To_apiserver_EgressSelectorConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error {
+	out.EgressSelections = *(*[]EgressSelection)(unsafe.Pointer(&in.EgressSelections))
+	return nil
+}
+
+// Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration is an autogenerated conversion function.
+func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in *apiserver.EgressSelectorConfiguration, out *EgressSelectorConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CABundle = in.CABundle
+	out.ClientKey = in.ClientKey
+	out.ClientCert = in.ClientCert
+	return nil
+}
+
+// Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig is an autogenerated conversion function.
+func Convert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in *HTTPConnectConfig, out *apiserver.HTTPConnectConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha1_HTTPConnectConfig_To_apiserver_HTTPConnectConfig(in, out, s)
+}
+
+func autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CABundle = in.CABundle
+	out.ClientKey = in.ClientKey
+	out.ClientCert = in.ClientCert
+	return nil
+}
+
+// Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig is an autogenerated conversion function.
+func Convert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in *apiserver.HTTPConnectConfig, out *HTTPConnectConfig, s conversion.Scope) error {
+	return autoConvert_apiserver_HTTPConnectConfig_To_v1alpha1_HTTPConnectConfig(in, out, s)
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,164 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdmissionConfiguration) DeepCopyInto(out *AdmissionConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.Plugins != nil {
+		in, out := &in.Plugins, &out.Plugins
+		*out = make([]AdmissionPluginConfiguration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionConfiguration.
+func (in *AdmissionConfiguration) DeepCopy() *AdmissionConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AdmissionConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AdmissionConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdmissionPluginConfiguration) DeepCopyInto(out *AdmissionPluginConfiguration) {
+	*out = *in
+	if in.Configuration != nil {
+		in, out := &in.Configuration, &out.Configuration
+		*out = new(runtime.Unknown)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdmissionPluginConfiguration.
+func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AdmissionPluginConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Connection) DeepCopyInto(out *Connection) {
+	*out = *in
+	if in.HTTPConnect != nil {
+		in, out := &in.HTTPConnect, &out.HTTPConnect
+		*out = new(HTTPConnectConfig)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Connection.
+func (in *Connection) DeepCopy() *Connection {
+	if in == nil {
+		return nil
+	}
+	out := new(Connection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelection) DeepCopyInto(out *EgressSelection) {
+	*out = *in
+	in.Connection.DeepCopyInto(&out.Connection)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelection.
+func (in *EgressSelection) DeepCopy() *EgressSelection {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelection)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *EgressSelectorConfiguration) DeepCopyInto(out *EgressSelectorConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.EgressSelections != nil {
+		in, out := &in.EgressSelections, &out.EgressSelections
+		*out = make([]EgressSelection, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressSelectorConfiguration.
+func (in *EgressSelectorConfiguration) DeepCopy() *EgressSelectorConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(EgressSelectorConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *HTTPConnectConfig) DeepCopyInto(out *HTTPConnectConfig) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPConnectConfig.
+func (in *HTTPConnectConfig) DeepCopy() *HTTPConnectConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(HTTPConnectConfig)
+	in.DeepCopyInto(out)
+	return out
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.defaults.go

@@ -0,0 +1,32 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by defaulter-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// RegisterDefaults adds defaulters functions to the given scheme.
+// Public to allow building arbitrary schemes.
+// All generated defaulters are covering - they call all nested defaulters.
+func RegisterDefaults(scheme *runtime.Scheme) error {
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -0,0 +1,176 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	"k8s.io/utils/path"
+	"sigs.k8s.io/yaml"
+)
+
+var cfgScheme = runtime.NewScheme()
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path.
+// It returns the loaded egress selector configuration if the input file aligns with the required syntax.
+// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op.
+// It does this by returning a nil configuration, which preserves backward compatibility.
+// This works because prior to this there was no egress selector configuration.
+// It returns an error if the file did not exist.
+func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) {
+	if configFilePath == "" {
+		return nil, nil
+	}
+	// a file was provided, so we just read it.
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
+	}
+	var decodedConfig v1alpha1.EgressSelectorConfiguration
+	err = yaml.Unmarshal(data, &decodedConfig)
+	if err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	if decodedConfig.Kind != "EgressSelectorConfiguration" {
+		return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind)
+	}
+	internalConfig := &apiserver.EgressSelectorConfiguration{}
+	if err := cfgScheme.Convert(&decodedConfig, internalConfig, nil); err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	return internalConfig, nil
+}
+
+// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for
+// common configuration errors. It will return error for problems such as configuring mtls/cert
+// settings for protocol which do not support security. It will also try to catch errors such as
+// incorrect file paths. It will return nil if it does not find anything wrong.
+func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if config == nil {
+		return allErrs // Treating a nil configuration as valid
+	}
+	for _, service := range config.EgressSelections {
+		base := field.NewPath("service", "connection")
+		switch service.Connection.Type {
+		case "direct":
+			allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...)
+		case "http-connect":
+			allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...)
+		default:
+			allErrs = append(allErrs, field.NotSupported(
+				base.Child("type"),
+				service.Connection.Type,
+				[]string{"direct", "http-connect"}))
+		}
+	}
+
+	return allErrs
+}
+
+func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	if connection.HTTPConnect != nil {
+		return field.ErrorList{field.Invalid(
+			fldPath.Child("httpConnect"),
+			"direct",
+			"httpConnect config should be absent for direct connect"),
+		}
+	}
+	return nil
+}
+
+func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if connection.HTTPConnect == nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect"),
+			"nil",
+			"httpConnect config should be present for http-connect"))
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") {
+		if connection.HTTPConnect.CABundle == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				"nil",
+				"http-connect via https requires caBundle"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect ca bundle does not exist"))
+		}
+		if connection.HTTPConnect.ClientCert == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				"nil",
+				"http-connect via https requires clientCert"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect client cert does not exist"))
+		}
+		if connection.HTTPConnect.ClientKey == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				"nil",
+				"http-connect via https requires clientKey"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect client key does not exist"))
+		}
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") {
+		if connection.HTTPConnect.CABundle != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect via http does not support caBundle"))
+		}
+		if connection.HTTPConnect.ClientCert != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect via http does not support clientCert"))
+		}
+		if connection.HTTPConnect.ClientKey != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect via http does not support clientKey"))
+		}
+	} else {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect", "url"),
+			connection.HTTPConnect.URL,
+			"supported connection protocols are http:// and https://"))
+	}
+	return allErrs
+}

vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/klog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
+
+// EgressSelector is the map of network context type to context dialer, for network egress.
+type EgressSelector struct {
+	egressToDialer map[EgressType]utilnet.DialFunc
+}
+
+// EgressType is an indicator of which egress selection should be used for sending traffic.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context
+type EgressType int
+
+const (
+	// Master is the EgressType for traffic intended to go to the control plane.
+	Master EgressType = iota
+	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
+	Etcd
+	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
+	Cluster
+)
+
+// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
+type NetworkContext struct {
+	// EgressSelectionName is the unique name of the
+	// EgressSelectorConfiguration which determines
+	// the network we route the traffic to.
+	EgressSelectionName EgressType
+}
+
+// Lookup is the interface to get the dialer function for the network context.
+type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
+
+// String returns the canonical string representation of the egress type
+func (s EgressType) String() string {
+	switch s {
+	case Master:
+		return "master"
+	case Etcd:
+		return "etcd"
+	case Cluster:
+		return "cluster"
+	default:
+		return "invalid"
+	}
+}
+
+// AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
+func (s EgressType) AsNetworkContext() NetworkContext {
+	return NetworkContext{EgressSelectionName: s}
+}
+
+func lookupServiceName(name string) (EgressType, error) {
+	switch strings.ToLower(name) {
+	case "master":
+		return Master, nil
+	case "etcd":
+		return Etcd, nil
+	case "cluster":
+		return Cluster, nil
+	}
+	return -1, fmt.Errorf("unrecognized service name %s", name)
+}
+
+func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) {
+	clientCert := connectConfig.ClientCert
+	clientKey := connectConfig.ClientKey
+	caCert := connectConfig.CABundle
+	proxyURL, err := url.Parse(connectConfig.URL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err)
+	}
+	proxyAddress := proxyURL.Host
+
+	clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
+	}
+	certPool := x509.NewCertPool()
+	certBytes, err := ioutil.ReadFile(caCert)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
+	}
+	ok := certPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		return nil, fmt.Errorf("failed to append CA cert to the cert pool")
+	}
+	contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		klog.V(4).Infof("Sending request to %q.", addr)
+		proxyConn, err := tls.Dial("tcp", proxyAddress,
+			&tls.Config{
+				Certificates: []tls.Certificate{clientCerts},
+				RootCAs:      certPool,
+			},
+		)
+		if err != nil {
+			return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err)
+		}
+		fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
+		br := bufio.NewReader(proxyConn)
+		res, err := http.ReadResponse(br, nil)
+		if err != nil {
+			proxyConn.Close()
+			return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
+				addr, proxyAddress, err)
+		}
+		if res.StatusCode != 200 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
+				proxyAddress, addr, res.StatusCode, res.Status)
+		}
+
+		// It's safe to discard the bufio.Reader here and return the
+		// original TCP conn directly because we only use this for
+		// TLS, and in TLS the client speaks first, so we know there's
+		// no unbuffered data. But we can double-check.
+		if br.Buffered() > 0 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
+				br.Buffered(), proxyAddress)
+		}
+		klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress)
+		return proxyConn, nil
+	}
+	return contextDialer, nil
+}
+
+// NewEgressSelector configures lookup mechanism for Lookup.
+// It does so based on a EgressSelectorConfiguration which was read at startup.
+func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
+	if config == nil || config.EgressSelections == nil {
+		// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
+		return nil, nil
+	}
+	cs := &EgressSelector{
+		egressToDialer: make(map[EgressType]utilnet.DialFunc),
+	}
+	for _, service := range config.EgressSelections {
+		name, err := lookupServiceName(service.Name)
+		if err != nil {
+			return nil, err
+		}
+		switch service.Connection.Type {
+		case "http-connect":
+			contextDialer, err := createConnectDialer(service.Connection.HTTPConnect)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create http-connect dialer: %v", err)
+			}
+			cs.egressToDialer[name] = contextDialer
+		case "direct":
+			cs.egressToDialer[name] = directDialer
+		default:
+			return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type)
+		}
+	}
+	return cs, nil
+}
+
+// Lookup gets the dialer function for the network context.
+// This is configured for the Kubernetes API Server at startup.
+func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
+	if cs.egressToDialer == nil {
+		// The round trip wrapper will over-ride the dialContext method appropriately
+		return nil, nil
+	}
+	return cs.egressToDialer[networkContext.EgressSelectionName], nil
+}

vendor/k8s.io/client-go/discovery/discovery_client.go

@@ -463,6 +463,13 @@ func setDiscoveryDefaults(config *restclient.Config) error {
 	if config.Timeout == 0 {
 		config.Timeout = defaultTimeout
 	}
+	if config.Burst == 0 && config.QPS < 100 {
+		// discovery is expected to be bursty, increase the default burst
+		// to accommodate looking up resource info for many API groups.
+		// matches burst set by ConfigFlags#ToDiscoveryClient().
+		// see https://issue.k8s.io/86149
+		config.Burst = 100
+	}
 	codec := runtime.NoopEncoder{Decoder: scheme.Codecs.UniversalDecoder()}
 	config.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec})
 	if len(config.UserAgent) == 0 {

vendor/k8s.io/client-go/tools/cache/reflector.go

@@ -74,9 +74,6 @@ type Reflector struct {
 	// observed when doing a sync with the underlying store
 	// it is thread safe, but not synchronized with the underlying store
 	lastSyncResourceVersion string
-	// isLastSyncResourceVersionGone is true if the previous list or watch request with lastSyncResourceVersion
-	// failed with an HTTP 410 (Gone) status code.
-	isLastSyncResourceVersionGone bool
 	// lastSyncResourceVersionMutex guards read/write access to lastSyncResourceVersion
 	lastSyncResourceVersionMutex sync.RWMutex
 	// WatchListPageSize is the requested chunk size of initial and resync watch lists.
@@ -188,7 +185,10 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
 	klog.V(3).Infof("Listing and watching %v from %s", r.expectedTypeName, r.name)
 	var resourceVersion string
 
-	options := metav1.ListOptions{ResourceVersion: r.relistResourceVersion()}
+	// Explicitly set "0" as resource version - it's fine for the List()
+	// to be served from cache and potentially be delayed relative to
+	// etcd contents. Reflector framework will catch up via Watch() eventually.
+	options := metav1.ListOptions{ResourceVersion: "0"}
 
 	if err := func() error {
 		initTrace := trace.New("Reflector ListAndWatch", trace.Field{"name", r.name})
@@ -211,17 +211,8 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
 			if r.WatchListPageSize != 0 {
 				pager.PageSize = r.WatchListPageSize
 			}
-
+			// Pager falls back to full list if paginated list calls fail due to an "Expired" error.
 			list, err = pager.List(context.Background(), options)
-			if isExpiredError(err) {
-				r.setIsLastSyncResourceVersionExpired(true)
-				// Retry immediately if the resource version used to list is expired.
-				// The pager already falls back to full list if paginated list calls fail due to an "Expired" error on
-				// continuation pages, but the pager might not be enabled, or the full list might fail because the
-				// resource version it is listing at is expired, so we need to fallback to resourceVersion="" in all
-				// to recover and ensure the reflector makes forward progress.
-				list, err = pager.List(context.Background(), metav1.ListOptions{ResourceVersion: r.relistResourceVersion()})
-			}
 			close(listCh)
 		}()
 		select {
@@ -234,7 +225,6 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
 		if err != nil {
 			return fmt.Errorf("%s: Failed to list %v: %v", r.name, r.expectedTypeName, err)
 		}
-		r.setIsLastSyncResourceVersionExpired(false) // list was successful
 		initTrace.Step("Objects listed")
 		listMetaInterface, err := meta.ListAccessor(list)
 		if err != nil {
@@ -308,13 +298,10 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
 
 		w, err := r.listerWatcher.Watch(options)
 		if err != nil {
-			switch {
-			case isExpiredError(err):
-				r.setIsLastSyncResourceVersionExpired(true)
-				klog.V(4).Infof("%s: watch of %v closed with: %v", r.name, r.expectedTypeName, err)
-			case err == io.EOF:
+			switch err {
+			case io.EOF:
 				// watch closed normally
-			case err == io.ErrUnexpectedEOF:
+			case io.ErrUnexpectedEOF:
 				klog.V(1).Infof("%s: Watch for %v closed with unexpected EOF: %v", r.name, r.expectedTypeName, err)
 			default:
 				utilruntime.HandleError(fmt.Errorf("%s: Failed to watch %v: %v", r.name, r.expectedTypeName, err))
@@ -333,8 +320,7 @@ func (r *Reflector) ListAndWatch(stopCh <-chan struct{}) error {
 		if err := r.watchHandler(w, &resourceVersion, resyncerrc, stopCh); err != nil {
 			if err != errorStopRequested {
 				switch {
-				case isExpiredError(err):
-					r.setIsLastSyncResourceVersionExpired(true)
+				case apierrs.IsResourceExpired(err):
 					klog.V(4).Infof("%s: watch of %v ended with: %v", r.name, r.expectedTypeName, err)
 				default:
 					klog.Warningf("%s: watch of %v ended with: %v", r.name, r.expectedTypeName, err)
@@ -446,42 +432,3 @@ func (r *Reflector) setLastSyncResourceVersion(v string) {
 	defer r.lastSyncResourceVersionMutex.Unlock()
 	r.lastSyncResourceVersion = v
 }
-
-// relistResourceVersion determines the resource version the reflector should list or relist from.
-// Returns either the lastSyncResourceVersion so that this reflector will relist with a resource
-// versions no older than has already been observed in relist results or watch events, or, if the last relist resulted
-// in an HTTP 410 (Gone) status code, returns "" so that the relist will use the latest resource version available in
-// etcd via a quorum read.
-func (r *Reflector) relistResourceVersion() string {
-	r.lastSyncResourceVersionMutex.RLock()
-	defer r.lastSyncResourceVersionMutex.RUnlock()
-
-	if r.isLastSyncResourceVersionGone {
-		// Since this reflector makes paginated list requests, and all paginated list requests skip the watch cache
-		// if the lastSyncResourceVersion is expired, we set ResourceVersion="" and list again to re-establish reflector
-		// to the latest available ResourceVersion, using a consistent read from etcd.
-		return ""
-	}
-	if r.lastSyncResourceVersion == "" {
-		// For performance reasons, initial list performed by reflector uses "0" as resource version to allow it to
-		// be served from the watch cache if it is enabled.
-		return "0"
-	}
-	return r.lastSyncResourceVersion
-}
-
-// setIsLastSyncResourceVersionExpired sets if the last list or watch request with lastSyncResourceVersion returned a
-// expired error: HTTP 410 (Gone) Status Code.
-func (r *Reflector) setIsLastSyncResourceVersionExpired(isExpired bool) {
-	r.lastSyncResourceVersionMutex.Lock()
-	defer r.lastSyncResourceVersionMutex.Unlock()
-	r.isLastSyncResourceVersionGone = isExpired
-}
-
-func isExpiredError(err error) bool {
-	// In Kubernetes 1.17 and earlier, the api server returns both apierrs.StatusReasonExpired and
-	// apierrs.StatusReasonGone for HTTP 410 (Gone) status code responses. In 1.18 the kube server is more consistent
-	// and always returns apierrs.StatusReasonExpired. For backward compatibility we can only remove the apierrs.IsGone
-	// check when we fully drop support for Kubernetes 1.17 servers from reflectors.
-	return apierrs.IsResourceExpired(err) || apierrs.IsGone(err)
-}

vendor/k8s.io/client-go/tools/clientcmd/api/v1/conversion.go

@@ -31,6 +31,9 @@ func Convert_Slice_v1_NamedCluster_To_Map_string_To_Pointer_api_Cluster(in *[]Na
 		if err := Convert_v1_Cluster_To_api_Cluster(&curr.Cluster, newCluster, s); err != nil {
 			return err
 		}
+		if *out == nil {
+			*out = make(map[string]*api.Cluster)
+		}
 		if (*out)[curr.Name] == nil {
 			(*out)[curr.Name] = newCluster
 		} else {
@@ -65,6 +68,9 @@ func Convert_Slice_v1_NamedAuthInfo_To_Map_string_To_Pointer_api_AuthInfo(in *[]
 		if err := Convert_v1_AuthInfo_To_api_AuthInfo(&curr.AuthInfo, newAuthInfo, s); err != nil {
 			return err
 		}
+		if *out == nil {
+			*out = make(map[string]*api.AuthInfo)
+		}
 		if (*out)[curr.Name] == nil {
 			(*out)[curr.Name] = newAuthInfo
 		} else {
@@ -99,6 +105,9 @@ func Convert_Slice_v1_NamedContext_To_Map_string_To_Pointer_api_Context(in *[]Na
 		if err := Convert_v1_Context_To_api_Context(&curr.Context, newContext, s); err != nil {
 			return err
 		}
+		if *out == nil {
+			*out = make(map[string]*api.Context)
+		}
 		if (*out)[curr.Name] == nil {
 			(*out)[curr.Name] = newContext
 		} else {
@@ -133,6 +142,9 @@ func Convert_Slice_v1_NamedExtension_To_Map_string_To_runtime_Object(in *[]Named
 		if err := runtime.Convert_runtime_RawExtension_To_runtime_Object(&curr.Extension, &newExtension, s); err != nil {
 			return err
 		}
+		if *out == nil {
+			*out = make(map[string]runtime.Object)
+		}
 		if (*out)[curr.Name] == nil {
 			(*out)[curr.Name] = newExtension
 		} else {

vendor/k8s.io/kubernetes/pkg/apis/storage/v1/util/helpers.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// IsDefaultStorageClassAnnotation represents a StorageClass annotation that
+// marks a class as the default StorageClass
+const IsDefaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class"
+
+// BetaIsDefaultStorageClassAnnotation is the beta version of IsDefaultStorageClassAnnotation.
+// TODO: remove Beta when no longer used
+const BetaIsDefaultStorageClassAnnotation = "storageclass.beta.kubernetes.io/is-default-class"
+
+// IsDefaultAnnotationText returns a pretty Yes/No String if
+// the annotation is set
+// TODO: remove Beta when no longer needed
+func IsDefaultAnnotationText(obj metav1.ObjectMeta) string {
+	if obj.Annotations[IsDefaultStorageClassAnnotation] == "true" {
+		return "Yes"
+	}
+	if obj.Annotations[BetaIsDefaultStorageClassAnnotation] == "true" {
+		return "Yes"
+	}
+
+	return "No"
+}
+
+// IsDefaultAnnotation returns a boolean if
+// the annotation is set
+// TODO: remove Beta when no longer needed
+func IsDefaultAnnotation(obj metav1.ObjectMeta) bool {
+	if obj.Annotations[IsDefaultStorageClassAnnotation] == "true" {
+		return true
+	}
+	if obj.Annotations[BetaIsDefaultStorageClassAnnotation] == "true" {
+		return true
+	}
+
+	return false
+}

vendor/k8s.io/kubernetes/pkg/volume/util/subpath/subpath_linux.go

@@ -240,7 +240,8 @@ func doCleanSubPaths(mounter mount.Interface, podDir string, volumeName string)
 				return err
 			}
 
-			if info.IsDir() {
+			// We need to check that info is not nil. This may happen when the incoming err is not nil due to stale mounts or permission errors.
+			if info != nil && info.IsDir() {
 				// skip subdirs of the volume: it only matters the first level to unmount, otherwise it would try to unmount subdir of the volume
 				return filepath.SkipDir
 			}

vendor/k8s.io/kubernetes/test/e2e/framework/config/config.go

@@ -0,0 +1,263 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package config simplifies the declaration of configuration options.
+// Right now the implementation maps them directly to command line
+// flags. When combined with test/e2e/framework/viperconfig in a test
+// suite, those flags then can also be read from a config file.
+//
+// The command line flags all get stored in a private flag set. The
+// developer of the E2E test suite decides how they are exposed. Options
+// include:
+// - exposing as normal flags in the actual command line:
+//   CopyFlags(Flags, flag.CommandLine)
+// - populate via test/e2e/framework/viperconfig:
+//   viperconfig.ViperizeFlags("my-config.yaml", "", Flags)
+// - a combination of both:
+//   CopyFlags(Flags, flag.CommandLine)
+//   viperconfig.ViperizeFlags("my-config.yaml", "", flag.CommandLine)
+//
+// Instead of defining flags one-by-one, test developers annotate a
+// structure with tags and then call a single function. This is the
+// same approach as in https://godoc.org/github.com/jessevdk/go-flags,
+// but implemented so that a test suite can continue to use the normal
+// "flag" package.
+//
+// For example, a file storage/csi.go might define:
+//
+//     var scaling struct {
+//             NumNodes int  `default:"1" description:"number of nodes to run on"`
+//             Master string
+//     }
+//     _ = config.AddOptions(&scaling, "storage.csi.scaling")
+//
+// This defines the following command line flags:
+//
+//     -storage.csi.scaling.numNodes=<int>  - number of nodes to run on (default: 1)
+//     -storage.csi.scaling.master=<string>
+//
+// All fields in the structure must be exported and have one of the following
+// types (same as in the `flag` package):
+// - bool
+// - time.Duration
+// - float64
+// - string
+// - int
+// - int64
+// - uint
+// - uint64
+// - and/or nested or embedded structures containing those basic types.
+//
+// Each basic entry may have a tag with these optional keys:
+//
+//     usage:   additional explanation of the option
+//     default: the default value, in the same format as it would
+//              be given on the command line and true/false for
+//              a boolean
+//
+// The names of the final configuration options are a combination of an
+// optional common prefix for all options in the structure and the
+// name of the fields, concatenated with a dot. To get names that are
+// consistent with the command line flags defined by `ginkgo`, the
+// initial character of each field name is converted to lower case.
+//
+// There is currently no support for aliases, so renaming the fields
+// or the common prefix will be visible to users of the test suite and
+// may breaks scripts which use the old names.
+//
+// The variable will be filled with the actual values by the test
+// suite before running tests. Beware that the code which registers
+// Ginkgo tests cannot use those config options, because registering
+// tests and options both run before the E2E test suite handles
+// parameters.
+package config
+
+import (
+	"flag"
+	"fmt"
+	"reflect"
+	"strconv"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Flags is the flag set that AddOptions adds to. Test authors should
+// also use it instead of directly adding to the global command line.
+var Flags = flag.NewFlagSet("", flag.ContinueOnError)
+
+// CopyFlags ensures that all flags that are defined in the source flag
+// set appear in the target flag set as if they had been defined there
+// directly. From the flag package it inherits the behavior that there
+// is a panic if the target already contains a flag from the source.
+func CopyFlags(source *flag.FlagSet, target *flag.FlagSet) {
+	source.VisitAll(func(flag *flag.Flag) {
+		// We don't need to copy flag.DefValue. The original
+		// default (from, say, flag.String) was stored in
+		// the value and gets extracted by Var for the help
+		// message.
+		target.Var(flag.Value, flag.Name, flag.Usage)
+	})
+}
+
+// AddOptions analyzes the options value and creates the necessary
+// flags to populate it.
+//
+// The prefix can be used to root the options deeper in the overall
+// set of options, with a dot separating different levels.
+//
+// The function always returns true, to enable this simplified
+// registration of options:
+// _ = AddOptions(...)
+//
+// It panics when it encounters an error, like unsupported types
+// or option name conflicts.
+func AddOptions(options interface{}, prefix string) bool {
+	return AddOptionsToSet(Flags, options, prefix)
+}
+
+// AddOptionsToSet is the same as AddOption, except that it allows choosing the flag set.
+func AddOptionsToSet(flags *flag.FlagSet, options interface{}, prefix string) bool {
+	optionsType := reflect.TypeOf(options)
+	if optionsType == nil {
+		panic("options parameter without a type - nil?!")
+	}
+	if optionsType.Kind() != reflect.Ptr || optionsType.Elem().Kind() != reflect.Struct {
+		panic(fmt.Sprintf("need a pointer to a struct, got instead: %T", options))
+	}
+	addStructFields(flags, optionsType.Elem(), reflect.Indirect(reflect.ValueOf(options)), prefix)
+	return true
+}
+
+func addStructFields(flags *flag.FlagSet, structType reflect.Type, structValue reflect.Value, prefix string) {
+	for i := 0; i < structValue.NumField(); i++ {
+		entry := structValue.Field(i)
+		addr := entry.Addr()
+		structField := structType.Field(i)
+		name := structField.Name
+		r, n := utf8.DecodeRuneInString(name)
+		name = string(unicode.ToLower(r)) + name[n:]
+		usage := structField.Tag.Get("usage")
+		def := structField.Tag.Get("default")
+		if prefix != "" {
+			name = prefix + "." + name
+		}
+		if structField.PkgPath != "" {
+			panic(fmt.Sprintf("struct entry %q not exported", name))
+		}
+		ptr := addr.Interface()
+		if structField.Anonymous {
+			// Entries in embedded fields are treated like
+			// entries, in the struct itself, i.e. we add
+			// them with the same prefix.
+			addStructFields(flags, structField.Type, entry, prefix)
+			continue
+		}
+		if structField.Type.Kind() == reflect.Struct {
+			// Add nested options.
+			addStructFields(flags, structField.Type, entry, name)
+			continue
+		}
+		// We could switch based on structField.Type. Doing a
+		// switch after getting an interface holding the
+		// pointer to the entry has the advantage that we
+		// immediately have something that we can add as flag
+		// variable.
+		//
+		// Perhaps generics will make this entire switch redundant someday...
+		switch ptr := ptr.(type) {
+		case *bool:
+			var defValue bool
+			parseDefault(&defValue, name, def)
+			flags.BoolVar(ptr, name, defValue, usage)
+		case *time.Duration:
+			var defValue time.Duration
+			parseDefault(&defValue, name, def)
+			flags.DurationVar(ptr, name, defValue, usage)
+		case *float64:
+			var defValue float64
+			parseDefault(&defValue, name, def)
+			flags.Float64Var(ptr, name, defValue, usage)
+		case *string:
+			flags.StringVar(ptr, name, def, usage)
+		case *int:
+			var defValue int
+			parseDefault(&defValue, name, def)
+			flags.IntVar(ptr, name, defValue, usage)
+		case *int64:
+			var defValue int64
+			parseDefault(&defValue, name, def)
+			flags.Int64Var(ptr, name, defValue, usage)
+		case *uint:
+			var defValue uint
+			parseDefault(&defValue, name, def)
+			flags.UintVar(ptr, name, defValue, usage)
+		case *uint64:
+			var defValue uint64
+			parseDefault(&defValue, name, def)
+			flags.Uint64Var(ptr, name, defValue, usage)
+		default:
+			panic(fmt.Sprintf("unsupported struct entry type %q: %T", name, entry.Interface()))
+		}
+	}
+}
+
+// parseDefault is necessary because "flag" wants the default in the
+// actual type and cannot take a string. It would be nice to reuse the
+// existing code for parsing from the "flag" package, but it isn't
+// exported.
+func parseDefault(value interface{}, name, def string) {
+	if def == "" {
+		return
+	}
+	checkErr := func(err error, value interface{}) {
+		if err != nil {
+			panic(fmt.Sprintf("invalid default %q for %T entry %s: %s", def, value, name, err))
+		}
+	}
+	switch value := value.(type) {
+	case *bool:
+		v, err := strconv.ParseBool(def)
+		checkErr(err, *value)
+		*value = v
+	case *time.Duration:
+		v, err := time.ParseDuration(def)
+		checkErr(err, *value)
+		*value = v
+	case *float64:
+		v, err := strconv.ParseFloat(def, 64)
+		checkErr(err, *value)
+		*value = v
+	case *int:
+		v, err := strconv.Atoi(def)
+		checkErr(err, *value)
+		*value = v
+	case *int64:
+		v, err := strconv.ParseInt(def, 0, 64)
+		checkErr(err, *value)
+		*value = v
+	case *uint:
+		v, err := strconv.ParseUint(def, 0, strconv.IntSize)
+		checkErr(err, *value)
+		*value = uint(v)
+	case *uint64:
+		v, err := strconv.ParseUint(def, 0, 64)
+		checkErr(err, *value)
+		*value = v
+	default:
+		panic(fmt.Sprintf("%q: setting defaults not supported for type %T", name, value))
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/pv/pv.go

@@ -0,0 +1,821 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	clientset "k8s.io/client-go/kubernetes"
+	storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util"
+	"k8s.io/kubernetes/pkg/volume/util"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+)
+
+const (
+	pdRetryTimeout  = 5 * time.Minute
+	pdRetryPollTime = 5 * time.Second
+
+	// PVBindingTimeout is how long PVs have to become bound.
+	PVBindingTimeout = 3 * time.Minute
+
+	// ClaimBindingTimeout is how long claims have to become bound.
+	ClaimBindingTimeout = 3 * time.Minute
+
+	// PVReclaimingTimeout is how long PVs have to beome reclaimed.
+	PVReclaimingTimeout = 3 * time.Minute
+
+	// PVDeletingTimeout is how long PVs have to become deleted.
+	PVDeletingTimeout = 3 * time.Minute
+
+	// VolumeSelectorKey is the key for volume selector.
+	VolumeSelectorKey = "e2e-pv-pool"
+)
+
+var (
+	// SELinuxLabel is common selinux labels.
+	SELinuxLabel = &v1.SELinuxOptions{
+		Level: "s0:c0,c1"}
+)
+
+type pvval struct{}
+
+// PVMap is a map of all PVs used in the multi pv-pvc tests. The key is the PV's name, which is
+// guaranteed to be unique. The value is {} (empty struct) since we're only interested
+// in the PV's name and if it is present. We must always Get the pv object before
+// referencing any of its values, eg its ClaimRef.
+type PVMap map[string]pvval
+
+type pvcval struct{}
+
+// PVCMap is a map of all PVCs used in the multi pv-pvc tests. The key is "namespace/pvc.Name". The
+// value is {} (empty struct) since we're only interested in the PVC's name and if it is
+// present. We must always Get the pvc object before referencing any of its values, eg.
+// its VolumeName.
+// Note: It's unsafe to add keys to a map in a loop. Their insertion in the map is
+//   unpredictable and can result in the same key being iterated over again.
+type PVCMap map[types.NamespacedName]pvcval
+
+// PersistentVolumeConfig is consumed by MakePersistentVolume() to generate a PV object
+// for varying storage options (NFS, ceph, glusterFS, etc.).
+// (+optional) prebind holds a pre-bound PVC
+// Example pvSource:
+//	pvSource: api.PersistentVolumeSource{
+//		NFS: &api.NFSVolumeSource{
+//	 		...
+//	 	},
+//	 }
+type PersistentVolumeConfig struct {
+	// [Optional] NamePrefix defaults to "pv-" if unset
+	NamePrefix string
+	// [Optional] Labels contains information used to organize and categorize
+	// objects
+	Labels labels.Set
+	// PVSource contains the details of the underlying volume and must be set
+	PVSource v1.PersistentVolumeSource
+	// [Optional] Prebind lets you specify a PVC to bind this PV to before
+	// creation
+	Prebind *v1.PersistentVolumeClaim
+	// [Optiona] ReclaimPolicy defaults to "Reclaim" if unset
+	ReclaimPolicy    v1.PersistentVolumeReclaimPolicy
+	StorageClassName string
+	// [Optional] NodeAffinity defines constraints that limit what nodes this
+	// volume can be accessed from.
+	NodeAffinity *v1.VolumeNodeAffinity
+	// [Optional] VolumeMode defaults to "Filesystem" if unset
+	VolumeMode *v1.PersistentVolumeMode
+	// [Optional] AccessModes defaults to RWO if unset
+	AccessModes []v1.PersistentVolumeAccessMode
+	// [Optional] Capacity is the storage capacity in Quantity format. Defaults
+	// to "2Gi" if unset
+	Capacity string
+}
+
+// PersistentVolumeClaimConfig is consumed by MakePersistentVolumeClaim() to
+// generate a PVC object.
+type PersistentVolumeClaimConfig struct {
+	// NamePrefix defaults to "pvc-" if unspecified
+	NamePrefix string
+	// ClaimSize must be specified in the Quantity format. Defaults to 2Gi if
+	// unspecified
+	ClaimSize string
+	// AccessModes defaults to RWO if unspecified
+	AccessModes      []v1.PersistentVolumeAccessMode
+	Annotations      map[string]string
+	Selector         *metav1.LabelSelector
+	StorageClassName *string
+	// VolumeMode defaults to nil if unspecified or specified as the empty
+	// string
+	VolumeMode *v1.PersistentVolumeMode
+}
+
+// PVPVCCleanup cleans up a pv and pvc in a single pv/pvc test case.
+// Note: delete errors are appended to []error so that we can attempt to delete both the pvc and pv.
+func PVPVCCleanup(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) []error {
+	var errs []error
+
+	if pvc != nil {
+		err := DeletePersistentVolumeClaim(c, pvc.Name, ns)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PVC %q: %v", pvc.Name, err))
+		}
+	} else {
+		e2elog.Logf("pvc is nil")
+	}
+	if pv != nil {
+		err := DeletePersistentVolume(c, pv.Name)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PV %q: %v", pv.Name, err))
+		}
+	} else {
+		e2elog.Logf("pv is nil")
+	}
+	return errs
+}
+
+// PVPVCMapCleanup Cleans up pvs and pvcs in multi-pv-pvc test cases. Entries found in the pv and claim maps are
+// deleted as long as the Delete api call succeeds.
+// Note: delete errors are appended to []error so that as many pvcs and pvs as possible are deleted.
+func PVPVCMapCleanup(c clientset.Interface, ns string, pvols PVMap, claims PVCMap) []error {
+	var errs []error
+
+	for pvcKey := range claims {
+		err := DeletePersistentVolumeClaim(c, pvcKey.Name, ns)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PVC %q: %v", pvcKey.Name, err))
+		} else {
+			delete(claims, pvcKey)
+		}
+	}
+
+	for pvKey := range pvols {
+		err := DeletePersistentVolume(c, pvKey)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to delete PV %q: %v", pvKey, err))
+		} else {
+			delete(pvols, pvKey)
+		}
+	}
+	return errs
+}
+
+// DeletePersistentVolume deletes the PV.
+func DeletePersistentVolume(c clientset.Interface, pvName string) error {
+	if c != nil && len(pvName) > 0 {
+		e2elog.Logf("Deleting PersistentVolume %q", pvName)
+		err := c.CoreV1().PersistentVolumes().Delete(pvName, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("PV Delete API error: %v", err)
+		}
+	}
+	return nil
+}
+
+// DeletePersistentVolumeClaim deletes the Claim.
+func DeletePersistentVolumeClaim(c clientset.Interface, pvcName string, ns string) error {
+	if c != nil && len(pvcName) > 0 {
+		e2elog.Logf("Deleting PersistentVolumeClaim %q", pvcName)
+		err := c.CoreV1().PersistentVolumeClaims(ns).Delete(pvcName, nil)
+		if err != nil && !apierrs.IsNotFound(err) {
+			return fmt.Errorf("PVC Delete API error: %v", err)
+		}
+	}
+	return nil
+}
+
+// DeletePVCandValidatePV deletes the PVC and waits for the PV to enter its expected phase. Validate that the PV
+// has been reclaimed (assumption here about reclaimPolicy). Caller tells this func which
+// phase value to expect for the pv bound to the to-be-deleted claim.
+func DeletePVCandValidatePV(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, expectPVPhase v1.PersistentVolumePhase) error {
+	pvname := pvc.Spec.VolumeName
+	e2elog.Logf("Deleting PVC %v to trigger reclamation of PV %v", pvc.Name, pvname)
+	err := DeletePersistentVolumeClaim(c, pvc.Name, ns)
+	if err != nil {
+		return err
+	}
+
+	// Wait for the PV's phase to return to be `expectPVPhase`
+	e2elog.Logf("Waiting for reclaim process to complete.")
+	err = WaitForPersistentVolumePhase(expectPVPhase, c, pv.Name, framework.Poll, PVReclaimingTimeout)
+	if err != nil {
+		return fmt.Errorf("pv %q phase did not become %v: %v", pv.Name, expectPVPhase, err)
+	}
+
+	// examine the pv's ClaimRef and UID and compare to expected values
+	pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PV Get API error: %v", err)
+	}
+	cr := pv.Spec.ClaimRef
+	if expectPVPhase == v1.VolumeAvailable {
+		if cr != nil && len(cr.UID) > 0 {
+			return fmt.Errorf("PV is 'Available' but ClaimRef.UID is not empty")
+		}
+	} else if expectPVPhase == v1.VolumeBound {
+		if cr == nil {
+			return fmt.Errorf("PV is 'Bound' but ClaimRef is nil")
+		}
+		if len(cr.UID) == 0 {
+			return fmt.Errorf("PV is 'Bound' but ClaimRef.UID is empty")
+		}
+	}
+
+	e2elog.Logf("PV %v now in %q phase", pv.Name, expectPVPhase)
+	return nil
+}
+
+// DeletePVCandValidatePVGroup wraps deletePVCandValidatePV() by calling the function in a loop over the PV map. Only bound PVs
+// are deleted. Validates that the claim was deleted and the PV is in the expected Phase (Released,
+// Available, Bound).
+// Note: if there are more claims than pvs then some of the remaining claims may bind to just made
+//   available pvs.
+func DeletePVCandValidatePVGroup(c clientset.Interface, ns string, pvols PVMap, claims PVCMap, expectPVPhase v1.PersistentVolumePhase) error {
+	var boundPVs, deletedPVCs int
+
+	for pvName := range pvols {
+		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("PV Get API error: %v", err)
+		}
+		cr := pv.Spec.ClaimRef
+		// if pv is bound then delete the pvc it is bound to
+		if cr != nil && len(cr.Name) > 0 {
+			boundPVs++
+			// Assert bound PVC is tracked in this test. Failing this might
+			// indicate external PVCs interfering with the test.
+			pvcKey := makePvcKey(ns, cr.Name)
+			if _, found := claims[pvcKey]; !found {
+				return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey)
+			}
+			// get the pvc for the delete call below
+			pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(cr.Name, metav1.GetOptions{})
+			if err == nil {
+				if err = DeletePVCandValidatePV(c, ns, pvc, pv, expectPVPhase); err != nil {
+					return err
+				}
+			} else if !apierrs.IsNotFound(err) {
+				return fmt.Errorf("PVC Get API error: %v", err)
+			}
+			// delete pvckey from map even if apierrs.IsNotFound above is true and thus the
+			// claim was not actually deleted here
+			delete(claims, pvcKey)
+			deletedPVCs++
+		}
+	}
+	if boundPVs != deletedPVCs {
+		return fmt.Errorf("expect number of bound PVs (%v) to equal number of deleted PVCs (%v)", boundPVs, deletedPVCs)
+	}
+	return nil
+}
+
+// create the PV resource. Fails test on error.
+func createPV(c clientset.Interface, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
+	pv, err := c.CoreV1().PersistentVolumes().Create(pv)
+	if err != nil {
+		return nil, fmt.Errorf("PV Create API error: %v", err)
+	}
+	return pv, nil
+}
+
+// CreatePV creates the PV resource. Fails test on error.
+func CreatePV(c clientset.Interface, pv *v1.PersistentVolume) (*v1.PersistentVolume, error) {
+	return createPV(c, pv)
+}
+
+// CreatePVC creates the PVC resource. Fails test on error.
+func CreatePVC(c clientset.Interface, ns string, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolumeClaim, error) {
+	pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Create(pvc)
+	if err != nil {
+		return nil, fmt.Errorf("PVC Create API error: %v", err)
+	}
+	return pvc, nil
+}
+
+// CreatePVCPV creates a PVC followed by the PV based on the passed in nfs-server ip and
+// namespace. If the "preBind" bool is true then pre-bind the PV to the PVC
+// via the PV's ClaimRef. Return the pv and pvc to reflect the created objects.
+// Note: in the pre-bind case the real PVC name, which is generated, is not
+//   known until after the PVC is instantiated. This is why the pvc is created
+//   before the pv.
+func CreatePVCPV(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) {
+	// make the pvc spec
+	pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+	preBindMsg := ""
+	if preBind {
+		preBindMsg = " pre-bound"
+		pvConfig.Prebind = pvc
+	}
+	// make the pv spec
+	pv := MakePersistentVolume(pvConfig)
+
+	ginkgo.By(fmt.Sprintf("Creating a PVC followed by a%s PV", preBindMsg))
+	pvc, err := CreatePVC(c, ns, pvc)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// instantiate the pv, handle pre-binding by ClaimRef if needed
+	if preBind {
+		pv.Spec.ClaimRef.Name = pvc.Name
+	}
+	pv, err = createPV(c, pv)
+	if err != nil {
+		return nil, pvc, err
+	}
+	return pv, pvc, nil
+}
+
+// CreatePVPVC creates a PV followed by the PVC based on the passed in nfs-server ip and
+// namespace. If the "preBind" bool is true then pre-bind the PVC to the PV
+// via the PVC's VolumeName. Return the pv and pvc to reflect the created
+// objects.
+// Note: in the pre-bind case the real PV name, which is generated, is not
+//   known until after the PV is instantiated. This is why the pv is created
+//   before the pvc.
+func CreatePVPVC(c clientset.Interface, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig, ns string, preBind bool) (*v1.PersistentVolume, *v1.PersistentVolumeClaim, error) {
+	preBindMsg := ""
+	if preBind {
+		preBindMsg = " pre-bound"
+	}
+	e2elog.Logf("Creating a PV followed by a%s PVC", preBindMsg)
+
+	// make the pv and pvc definitions
+	pv := MakePersistentVolume(pvConfig)
+	pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+
+	// instantiate the pv
+	pv, err := createPV(c, pv)
+	if err != nil {
+		return nil, nil, err
+	}
+	// instantiate the pvc, handle pre-binding by VolumeName if needed
+	if preBind {
+		pvc.Spec.VolumeName = pv.Name
+	}
+	pvc, err = CreatePVC(c, ns, pvc)
+	if err != nil {
+		return pv, nil, err
+	}
+	return pv, pvc, nil
+}
+
+// CreatePVsPVCs creates the desired number of PVs and PVCs and returns them in separate maps. If the
+// number of PVs != the number of PVCs then the min of those two counts is the number of
+// PVs expected to bind. If a Create error occurs, the returned maps may contain pv and pvc
+// entries for the resources that were successfully created. In other words, when the caller
+// sees an error returned, it needs to decide what to do about entries in the maps.
+// Note: when the test suite deletes the namespace orphaned pvcs and pods are deleted. However,
+//   orphaned pvs are not deleted and will remain after the suite completes.
+func CreatePVsPVCs(numpvs, numpvcs int, c clientset.Interface, ns string, pvConfig PersistentVolumeConfig, pvcConfig PersistentVolumeClaimConfig) (PVMap, PVCMap, error) {
+	pvMap := make(PVMap, numpvs)
+	pvcMap := make(PVCMap, numpvcs)
+	extraPVCs := 0
+	extraPVs := numpvs - numpvcs
+	if extraPVs < 0 {
+		extraPVCs = -extraPVs
+		extraPVs = 0
+	}
+	pvsToCreate := numpvs - extraPVs // want the min(numpvs, numpvcs)
+
+	// create pvs and pvcs
+	for i := 0; i < pvsToCreate; i++ {
+		pv, pvc, err := CreatePVPVC(c, pvConfig, pvcConfig, ns, false)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvMap[pv.Name] = pvval{}
+		pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
+	}
+
+	// create extra pvs or pvcs as needed
+	for i := 0; i < extraPVs; i++ {
+		pv := MakePersistentVolume(pvConfig)
+		pv, err := createPV(c, pv)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvMap[pv.Name] = pvval{}
+	}
+	for i := 0; i < extraPVCs; i++ {
+		pvc := MakePersistentVolumeClaim(pvcConfig, ns)
+		pvc, err := CreatePVC(c, ns, pvc)
+		if err != nil {
+			return pvMap, pvcMap, err
+		}
+		pvcMap[makePvcKey(ns, pvc.Name)] = pvcval{}
+	}
+	return pvMap, pvcMap, nil
+}
+
+// WaitOnPVandPVC waits for the pv and pvc to bind to each other.
+func WaitOnPVandPVC(c clientset.Interface, ns string, pv *v1.PersistentVolume, pvc *v1.PersistentVolumeClaim) error {
+	// Wait for newly created PVC to bind to the PV
+	e2elog.Logf("Waiting for PV %v to bind to PVC %v", pv.Name, pvc.Name)
+	err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, framework.Poll, ClaimBindingTimeout)
+	if err != nil {
+		return fmt.Errorf("PVC %q did not become Bound: %v", pvc.Name, err)
+	}
+
+	// Wait for PersistentVolume.Status.Phase to be Bound, which it should be
+	// since the PVC is already bound.
+	err = WaitForPersistentVolumePhase(v1.VolumeBound, c, pv.Name, framework.Poll, PVBindingTimeout)
+	if err != nil {
+		return fmt.Errorf("PV %q did not become Bound: %v", pv.Name, err)
+	}
+
+	// Re-get the pv and pvc objects
+	pv, err = c.CoreV1().PersistentVolumes().Get(pv.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PV Get API error: %v", err)
+	}
+	pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("PVC Get API error: %v", err)
+	}
+
+	// The pv and pvc are both bound, but to each other?
+	// Check that the PersistentVolume.ClaimRef matches the PVC
+	if pv.Spec.ClaimRef == nil {
+		return fmt.Errorf("PV %q ClaimRef is nil", pv.Name)
+	}
+	if pv.Spec.ClaimRef.Name != pvc.Name {
+		return fmt.Errorf("PV %q ClaimRef's name (%q) should be %q", pv.Name, pv.Spec.ClaimRef.Name, pvc.Name)
+	}
+	if pvc.Spec.VolumeName != pv.Name {
+		return fmt.Errorf("PVC %q VolumeName (%q) should be %q", pvc.Name, pvc.Spec.VolumeName, pv.Name)
+	}
+	if pv.Spec.ClaimRef.UID != pvc.UID {
+		return fmt.Errorf("PV %q ClaimRef's UID (%q) should be %q", pv.Name, pv.Spec.ClaimRef.UID, pvc.UID)
+	}
+	return nil
+}
+
+// WaitAndVerifyBinds searches for bound PVs and PVCs by examining pvols for non-nil claimRefs.
+// NOTE: Each iteration waits for a maximum of 3 minutes per PV and, if the PV is bound,
+//   up to 3 minutes for the PVC. When the number of PVs != number of PVCs, this can lead
+//   to situations where the maximum wait times are reached several times in succession,
+//   extending test time. Thus, it is recommended to keep the delta between PVs and PVCs
+//   small.
+func WaitAndVerifyBinds(c clientset.Interface, ns string, pvols PVMap, claims PVCMap, testExpected bool) error {
+	var actualBinds int
+	expectedBinds := len(pvols)
+	if expectedBinds > len(claims) { // want the min of # pvs or #pvcs
+		expectedBinds = len(claims)
+	}
+
+	for pvName := range pvols {
+		err := WaitForPersistentVolumePhase(v1.VolumeBound, c, pvName, framework.Poll, PVBindingTimeout)
+		if err != nil && len(pvols) > len(claims) {
+			e2elog.Logf("WARN: pv %v is not bound after max wait", pvName)
+			e2elog.Logf("      This may be ok since there are more pvs than pvcs")
+			continue
+		}
+		if err != nil {
+			return fmt.Errorf("PV %q did not become Bound: %v", pvName, err)
+		}
+
+		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("PV Get API error: %v", err)
+		}
+		cr := pv.Spec.ClaimRef
+		if cr != nil && len(cr.Name) > 0 {
+			// Assert bound pvc is a test resource. Failing assertion could
+			// indicate non-test PVC interference or a bug in the test
+			pvcKey := makePvcKey(ns, cr.Name)
+			if _, found := claims[pvcKey]; !found {
+				return fmt.Errorf("internal: claims map is missing pvc %q", pvcKey)
+			}
+
+			err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, cr.Name, framework.Poll, ClaimBindingTimeout)
+			if err != nil {
+				return fmt.Errorf("PVC %q did not become Bound: %v", cr.Name, err)
+			}
+			actualBinds++
+		}
+	}
+
+	if testExpected && actualBinds != expectedBinds {
+		return fmt.Errorf("expect number of bound PVs (%v) to equal number of claims (%v)", actualBinds, expectedBinds)
+	}
+	return nil
+}
+
+// Return a pvckey struct.
+func makePvcKey(ns, name string) types.NamespacedName {
+	return types.NamespacedName{Namespace: ns, Name: name}
+}
+
+// MakePersistentVolume returns a PV definition based on the nfs server IP. If the PVC is not nil
+// then the PV is defined with a ClaimRef which includes the PVC's namespace.
+// If the PVC is nil then the PV is not defined with a ClaimRef.  If no reclaimPolicy
+// is assigned, assumes "Retain". Specs are expected to match the test's PVC.
+// Note: the passed-in claim does not have a name until it is created and thus the PV's
+//   ClaimRef cannot be completely filled-in in this func. Therefore, the ClaimRef's name
+//   is added later in CreatePVCPV.
+func MakePersistentVolume(pvConfig PersistentVolumeConfig) *v1.PersistentVolume {
+	var claimRef *v1.ObjectReference
+
+	if len(pvConfig.AccessModes) == 0 {
+		pvConfig.AccessModes = append(pvConfig.AccessModes, v1.ReadWriteOnce)
+	}
+
+	if len(pvConfig.NamePrefix) == 0 {
+		pvConfig.NamePrefix = "pv-"
+	}
+
+	if pvConfig.ReclaimPolicy == "" {
+		pvConfig.ReclaimPolicy = v1.PersistentVolumeReclaimRetain
+	}
+
+	if len(pvConfig.Capacity) == 0 {
+		pvConfig.Capacity = "2Gi"
+	}
+
+	if pvConfig.Prebind != nil {
+		claimRef = &v1.ObjectReference{
+			Kind:       "PersistentVolumeClaim",
+			APIVersion: "v1",
+			Name:       pvConfig.Prebind.Name,
+			Namespace:  pvConfig.Prebind.Namespace,
+			UID:        pvConfig.Prebind.UID,
+		}
+	}
+
+	return &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: pvConfig.NamePrefix,
+			Labels:       pvConfig.Labels,
+			Annotations: map[string]string{
+				util.VolumeGidAnnotationKey: "777",
+			},
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeReclaimPolicy: pvConfig.ReclaimPolicy,
+			Capacity: v1.ResourceList{
+				v1.ResourceStorage: resource.MustParse(pvConfig.Capacity),
+			},
+			PersistentVolumeSource: pvConfig.PVSource,
+			AccessModes:            pvConfig.AccessModes,
+			ClaimRef:               claimRef,
+			StorageClassName:       pvConfig.StorageClassName,
+			NodeAffinity:           pvConfig.NodeAffinity,
+			VolumeMode:             pvConfig.VolumeMode,
+		},
+	}
+}
+
+// MakePersistentVolumeClaim returns a PVC API Object based on the PersistentVolumeClaimConfig.
+func MakePersistentVolumeClaim(cfg PersistentVolumeClaimConfig, ns string) *v1.PersistentVolumeClaim {
+
+	if len(cfg.AccessModes) == 0 {
+		cfg.AccessModes = append(cfg.AccessModes, v1.ReadWriteOnce)
+	}
+
+	if len(cfg.ClaimSize) == 0 {
+		cfg.ClaimSize = "2Gi"
+	}
+
+	if len(cfg.NamePrefix) == 0 {
+		cfg.NamePrefix = "pvc-"
+	}
+
+	if cfg.VolumeMode != nil && *cfg.VolumeMode == "" {
+		e2elog.Logf("Warning: Making PVC: VolumeMode specified as invalid empty string, treating as nil")
+		cfg.VolumeMode = nil
+	}
+
+	return &v1.PersistentVolumeClaim{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: cfg.NamePrefix,
+			Namespace:    ns,
+			Annotations:  cfg.Annotations,
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			Selector:    cfg.Selector,
+			AccessModes: cfg.AccessModes,
+			Resources: v1.ResourceRequirements{
+				Requests: v1.ResourceList{
+					v1.ResourceStorage: resource.MustParse(cfg.ClaimSize),
+				},
+			},
+			StorageClassName: cfg.StorageClassName,
+			VolumeMode:       cfg.VolumeMode,
+		},
+	}
+}
+
+func createPDWithRetry(zone string) (string, error) {
+	var err error
+	var newDiskName string
+	for start := time.Now(); time.Since(start) < pdRetryTimeout; time.Sleep(pdRetryPollTime) {
+		newDiskName, err = createPD(zone)
+		if err != nil {
+			e2elog.Logf("Couldn't create a new PD, sleeping 5 seconds: %v", err)
+			continue
+		}
+		e2elog.Logf("Successfully created a new PD: %q.", newDiskName)
+		return newDiskName, nil
+	}
+	return "", err
+}
+
+// CreatePDWithRetry creates PD with retry.
+func CreatePDWithRetry() (string, error) {
+	return createPDWithRetry("")
+}
+
+// CreatePDWithRetryAndZone creates PD on zone with retry.
+func CreatePDWithRetryAndZone(zone string) (string, error) {
+	return createPDWithRetry(zone)
+}
+
+// DeletePDWithRetry deletes PD with retry.
+func DeletePDWithRetry(diskName string) error {
+	var err error
+	for start := time.Now(); time.Since(start) < pdRetryTimeout; time.Sleep(pdRetryPollTime) {
+		err = deletePD(diskName)
+		if err != nil {
+			e2elog.Logf("Couldn't delete PD %q, sleeping %v: %v", diskName, pdRetryPollTime, err)
+			continue
+		}
+		e2elog.Logf("Successfully deleted PD %q.", diskName)
+		return nil
+	}
+	return fmt.Errorf("unable to delete PD %q: %v", diskName, err)
+}
+
+func createPD(zone string) (string, error) {
+	if zone == "" {
+		zone = framework.TestContext.CloudConfig.Zone
+	}
+	return framework.TestContext.CloudConfig.Provider.CreatePD(zone)
+}
+
+func deletePD(pdName string) error {
+	return framework.TestContext.CloudConfig.Provider.DeletePD(pdName)
+}
+
+// MakeWritePod returns a pod definition based on the namespace. The pod references the PVC's
+// name.
+func MakeWritePod(ns string, pvc *v1.PersistentVolumeClaim) *v1.Pod {
+	return e2epod.MakePod(ns, nil, []*v1.PersistentVolumeClaim{pvc}, true, "touch /mnt/volume1/SUCCESS && (id -G | grep -E '\\b777\\b')")
+}
+
+// WaitForPVClaimBoundPhase waits until all pvcs phase set to bound
+func WaitForPVClaimBoundPhase(client clientset.Interface, pvclaims []*v1.PersistentVolumeClaim, timeout time.Duration) ([]*v1.PersistentVolume, error) {
+	persistentvolumes := make([]*v1.PersistentVolume, len(pvclaims))
+
+	for index, claim := range pvclaims {
+		err := WaitForPersistentVolumeClaimPhase(v1.ClaimBound, client, claim.Namespace, claim.Name, framework.Poll, timeout)
+		if err != nil {
+			return persistentvolumes, err
+		}
+		// Get new copy of the claim
+		claim, err = client.CoreV1().PersistentVolumeClaims(claim.Namespace).Get(claim.Name, metav1.GetOptions{})
+		if err != nil {
+			return persistentvolumes, fmt.Errorf("PVC Get API error: %v", err)
+		}
+		// Get the bounded PV
+		persistentvolumes[index], err = client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+		if err != nil {
+			return persistentvolumes, fmt.Errorf("PV Get API error: %v", err)
+		}
+	}
+	return persistentvolumes, nil
+}
+
+// WaitForPersistentVolumePhase waits for a PersistentVolume to be in a specific phase or until timeout occurs, whichever comes first.
+func WaitForPersistentVolumePhase(phase v1.PersistentVolumePhase, c clientset.Interface, pvName string, Poll, timeout time.Duration) error {
+	e2elog.Logf("Waiting up to %v for PersistentVolume %s to have phase %s", timeout, pvName, phase)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
+		pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
+		if err != nil {
+			e2elog.Logf("Get persistent volume %s in failed, ignoring for %v: %v", pvName, Poll, err)
+			continue
+		}
+		if pv.Status.Phase == phase {
+			e2elog.Logf("PersistentVolume %s found and phase=%s (%v)", pvName, phase, time.Since(start))
+			return nil
+		}
+		e2elog.Logf("PersistentVolume %s found but phase is %s instead of %s.", pvName, pv.Status.Phase, phase)
+	}
+	return fmt.Errorf("PersistentVolume %s not in phase %s within %v", pvName, phase, timeout)
+}
+
+// WaitForPersistentVolumeClaimPhase waits for a PersistentVolumeClaim to be in a specific phase or until timeout occurs, whichever comes first.
+func WaitForPersistentVolumeClaimPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcName string, Poll, timeout time.Duration) error {
+	return WaitForPersistentVolumeClaimsPhase(phase, c, ns, []string{pvcName}, Poll, timeout, true)
+}
+
+// WaitForPersistentVolumeClaimsPhase waits for any (if matchAny is true) or all (if matchAny is false) PersistentVolumeClaims
+// to be in a specific phase or until timeout occurs, whichever comes first.
+func WaitForPersistentVolumeClaimsPhase(phase v1.PersistentVolumeClaimPhase, c clientset.Interface, ns string, pvcNames []string, Poll, timeout time.Duration, matchAny bool) error {
+	if len(pvcNames) == 0 {
+		return fmt.Errorf("Incorrect parameter: Need at least one PVC to track. Found 0")
+	}
+	e2elog.Logf("Waiting up to %v for PersistentVolumeClaims %v to have phase %s", timeout, pvcNames, phase)
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(Poll) {
+		phaseFoundInAllClaims := true
+		for _, pvcName := range pvcNames {
+			pvc, err := c.CoreV1().PersistentVolumeClaims(ns).Get(pvcName, metav1.GetOptions{})
+			if err != nil {
+				e2elog.Logf("Failed to get claim %q, retrying in %v. Error: %v", pvcName, Poll, err)
+				continue
+			}
+			if pvc.Status.Phase == phase {
+				e2elog.Logf("PersistentVolumeClaim %s found and phase=%s (%v)", pvcName, phase, time.Since(start))
+				if matchAny {
+					return nil
+				}
+			} else {
+				e2elog.Logf("PersistentVolumeClaim %s found but phase is %s instead of %s.", pvcName, pvc.Status.Phase, phase)
+				phaseFoundInAllClaims = false
+			}
+		}
+		if phaseFoundInAllClaims {
+			return nil
+		}
+	}
+	return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout)
+}
+
+// CreatePVSource creates a PV source.
+func CreatePVSource(zone string) (*v1.PersistentVolumeSource, error) {
+	diskName, err := CreatePDWithRetryAndZone(zone)
+	if err != nil {
+		return nil, err
+	}
+	return framework.TestContext.CloudConfig.Provider.CreatePVSource(zone, diskName)
+}
+
+// DeletePVSource deletes a PV source.
+func DeletePVSource(pvSource *v1.PersistentVolumeSource) error {
+	return framework.TestContext.CloudConfig.Provider.DeletePVSource(pvSource)
+}
+
+// GetBoundPV returns a PV details.
+func GetBoundPV(client clientset.Interface, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) {
+	// Get new copy of the claim
+	claim, err := client.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Get the bound PV
+	pv, err := client.CoreV1().PersistentVolumes().Get(claim.Spec.VolumeName, metav1.GetOptions{})
+	return pv, err
+}
+
+// GetDefaultStorageClassName returns default storageClass or return error
+func GetDefaultStorageClassName(c clientset.Interface) (string, error) {
+	list, err := c.StorageV1().StorageClasses().List(metav1.ListOptions{})
+	if err != nil {
+		return "", fmt.Errorf("Error listing storage classes: %v", err)
+	}
+	var scName string
+	for _, sc := range list.Items {
+		if storageutil.IsDefaultAnnotation(sc.ObjectMeta) {
+			if len(scName) != 0 {
+				return "", fmt.Errorf("Multiple default storage classes found: %q and %q", scName, sc.Name)
+			}
+			scName = sc.Name
+		}
+	}
+	if len(scName) == 0 {
+		return "", fmt.Errorf("No default storage class found")
+	}
+	e2elog.Logf("Default storage class: %q", scName)
+	return scName, nil
+}
+
+// SkipIfNoDefaultStorageClass skips tests if no default SC can be found.
+func SkipIfNoDefaultStorageClass(c clientset.Interface) {
+	_, err := GetDefaultStorageClassName(c)
+	if err != nil {
+		framework.Skipf("error finding default storageClass : %v", err)
+	}
+}