build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>

e2e/vendor/k8s.io/apiserver/pkg/server/config_selfclient.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net"
+
+	restclient "k8s.io/client-go/rest"
+	netutils "k8s.io/utils/net"
+)
+
+// LoopbackClientServerNameOverride is passed to the apiserver from the loopback client in order to
+// select the loopback certificate via SNI if TLS is used.
+const LoopbackClientServerNameOverride = "apiserver-loopback-client"
+
+func (s *SecureServingInfo) NewClientConfig(caCert []byte) (*restclient.Config, error) {
+	if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) {
+		return nil, nil
+	}
+
+	host, port, err := LoopbackHostPort(s.Listener.Addr().String())
+	if err != nil {
+		return nil, err
+	}
+
+	return &restclient.Config{
+		// Do not limit loopback client QPS.
+		QPS:  -1,
+		Host: "https://" + net.JoinHostPort(host, port),
+		TLSClientConfig: restclient.TLSClientConfig{
+			CAData: caCert,
+		},
+	}, nil
+}
+
+func (s *SecureServingInfo) NewLoopbackClientConfig(token string, loopbackCert []byte) (*restclient.Config, error) {
+	c, err := s.NewClientConfig(loopbackCert)
+	if err != nil || c == nil {
+		return c, err
+	}
+
+	c.BearerToken = token
+	// override the ServerName to select our loopback certificate via SNI. This name is also
+	// used by the client to compare the returns server certificate against.
+	c.TLSClientConfig.ServerName = LoopbackClientServerNameOverride
+
+	return c, nil
+}
+
+// LoopbackHostPort returns the host and port loopback REST clients should use
+// to contact the server.
+func LoopbackHostPort(bindAddress string) (string, string, error) {
+	host, port, err := net.SplitHostPort(bindAddress)
+	if err != nil {
+		// should never happen
+		return "", "", fmt.Errorf("invalid server bind address: %q", bindAddress)
+	}
+
+	isIPv6 := netutils.IsIPv6String(host)
+
+	// Value is expected to be an IP or DNS name, not "0.0.0.0".
+	if host == "0.0.0.0" || host == "::" {
+		// Get ip of local interface, but fall back to "localhost".
+		// Note that "localhost" is resolved with the external nameserver first with Go's stdlib.
+		// So if localhost.<yoursearchdomain> resolves, we don't get a 127.0.0.1 as expected.
+		host = getLoopbackAddress(isIPv6)
+	}
+	return host, port, nil
+}
+
+// getLoopbackAddress returns the ip address of local loopback interface. If any error occurs or loopback interface is not found, will fall back to "localhost"
+func getLoopbackAddress(wantIPv6 bool) string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, address := range addrs {
+			if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && wantIPv6 == netutils.IsIPv6(ipnet.IP) {
+				return ipnet.IP.String()
+			}
+		}
+	}
+	return "localhost"
+}

e2e/vendor/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/klog/v2"
+)
+
+// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
+type resourceExpirationEvaluator struct {
+	currentVersion *apimachineryversion.Version
+	isAlpha        bool
+	// This is usually set for testing for which tests need to be removed.  This prevent insta-failing CI.
+	// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
+	strictRemovedHandlingInAlpha bool
+	// This is usually set by a cluster-admin looking for a short-term escape hatch after something bad happened.
+	// This should be made a flag before merge
+	// Set KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE to prevent removing APIs for one more release.
+	serveRemovedAPIsOneMoreRelease bool
+}
+
+// ResourceExpirationEvaluator indicates whether or not a resource should be served.
+type ResourceExpirationEvaluator interface {
+	// RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+	// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+	RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage)
+	// ShouldServeForVersion returns true if a particular version cut off is after the current version
+	ShouldServeForVersion(majorRemoved, minorRemoved int) bool
+}
+
+func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
+	if currentVersion == nil {
+		return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
+	}
+	klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
+	ret := &resourceExpirationEvaluator{
+		strictRemovedHandlingInAlpha: false,
+	}
+	// Only keeps the major and minor versions from input version.
+	ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
+	ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
+
+	if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
+		// do nothing
+	} else if envBool, err := strconv.ParseBool(envString); err != nil {
+		return nil, err
+	} else {
+		ret.strictRemovedHandlingInAlpha = envBool
+	}
+
+	if envString, ok := os.LookupEnv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE"); !ok {
+		// do nothing
+	} else if envBool, err := strconv.ParseBool(envString); err != nil {
+		return nil, err
+	} else {
+		ret.serveRemovedAPIsOneMoreRelease = envBool
+	}
+
+	return ret, nil
+}
+
+func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
+	internalPtr := resourceServingInfo.New()
+
+	target := gv
+	// honor storage that overrides group version (used for things like scale subresources)
+	if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
+		target = versionProvider.GroupVersionKind(target).GroupVersion()
+	}
+
+	versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return false
+	}
+
+	introduced, ok := versionedPtr.(introducedInterface)
+	if ok {
+		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+		verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+		if e.currentVersion.LessThan(verIntroduced) {
+			return false
+		}
+	}
+
+	removed, ok := versionedPtr.(removedInterface)
+	if !ok {
+		return true
+	}
+	majorRemoved, minorRemoved := removed.APILifecycleRemoved()
+	return e.ShouldServeForVersion(majorRemoved, minorRemoved)
+}
+
+func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
+	removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
+	if removedVer.GreaterThan(e.currentVersion) {
+		return true
+	}
+	if removedVer.LessThan(e.currentVersion) {
+		return false
+	}
+	// at this point major and minor are equal, so this API should be removed when the current release GAs.
+	// If this is an alpha tag, don't remove by default, but allow the option.
+	// If the cluster-admin has requested serving one more release, allow it.
+	if e.isAlpha && e.strictRemovedHandlingInAlpha { // don't serve in alpha if we want strict handling
+		return false
+	}
+	if e.isAlpha { // alphas are allowed to continue serving expired betas while we clean up the test
+		return true
+	}
+	if e.serveRemovedAPIsOneMoreRelease { // cluster-admins are allowed to kick the can one release down the road
+		return true
+	}
+	return false
+}
+
+type removedInterface interface {
+	APILifecycleRemoved() (major, minor int)
+}
+
+// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
+type introducedInterface interface {
+	APILifecycleIntroduced() (major, minor int)
+}
+
+// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
+	versionsToRemove := sets.NewString()
+	for apiVersion := range sets.StringKeySet(versionedResourcesStorageMap) {
+		versionToResource := versionedResourcesStorageMap[apiVersion]
+		resourcesToRemove := sets.NewString()
+		for resourceName, resourceServingInfo := range versionToResource {
+			if !e.shouldServe(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
+				resourcesToRemove.Insert(resourceName)
+			}
+		}
+
+		for resourceName := range versionedResourcesStorageMap[apiVersion] {
+			if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
+				continue
+			}
+
+			klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
+			storage := versionToResource[resourceName]
+			storage.Destroy()
+			delete(versionToResource, resourceName)
+		}
+		versionedResourcesStorageMap[apiVersion] = versionToResource
+
+		if len(versionedResourcesStorageMap[apiVersion]) == 0 {
+			versionsToRemove.Insert(apiVersion)
+		}
+	}
+
+	for _, apiVersion := range versionsToRemove.List() {
+		klog.V(1).Infof("Removing version %v.%v because it is time to stop serving it because it has no resources per APILifecycle.", apiVersion, groupName)
+		delete(versionedResourcesStorageMap, apiVersion)
+	}
+}
+
+func shouldRemoveResourceAndSubresources(resourcesToRemove sets.String, resourceName string) bool {
+	for _, resourceToRemove := range resourcesToRemove.List() {
+		if resourceName == resourceToRemove {
+			return true
+		}
+		// our API works on nesting, so you can have deployments, deployments/status, and deployments/scale.  Not all subresources
+		// serve the parent type, but if the parent type (deployments in this case), has been removed, it's subresources should be removed too.
+		if strings.HasPrefix(resourceName, resourceToRemove+"/") {
+			return true
+		}
+	}
+	return false
+}

e2e/vendor/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/rest"
+)
+
+// DeprecatedInsecureServingInfo is the main context object for the insecure http server.
+// HTTP does NOT include authentication or authorization.
+// You shouldn't be using this.  It makes sig-auth sad.
+type DeprecatedInsecureServingInfo struct {
+	// Listener is the secure server network listener.
+	Listener net.Listener
+	// optional server name for log messages
+	Name string
+}
+
+// Serve starts an insecure http server with the given handler. It fails only if
+// the initial listen call fails. It does not block.
+func (s *DeprecatedInsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error {
+	insecureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+
+		IdleTimeout:       90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+		ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound
+	}
+
+	if len(s.Name) > 0 {
+		klog.Infof("Serving %s insecurely on %s", s.Name, s.Listener.Addr())
+	} else {
+		klog.Infof("Serving insecurely on %s", s.Listener.Addr())
+	}
+	_, _, err := RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
+	// NOTE: we do not handle stoppedCh returned by RunServer for graceful termination here
+	return err
+}
+
+func (s *DeprecatedInsecureServingInfo) NewLoopbackClientConfig() (*rest.Config, error) {
+	if s == nil {
+		return nil, nil
+	}
+
+	host, port, err := LoopbackHostPort(s.Listener.Addr().String())
+	if err != nil {
+		return nil, err
+	}
+
+	return &rest.Config{
+		Host: "http://" + net.JoinHostPort(host, port),
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:   50,
+		Burst: 100,
+	}, nil
+}
+
+// InsecureSuperuser implements authenticator.Request to always return a superuser.
+// This is functionally equivalent to skipping authentication and authorization,
+// but allows apiserver code to stop special-casing a nil user to skip authorization checks.
+type InsecureSuperuser struct{}
+
+func (InsecureSuperuser) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
+	auds, _ := authenticator.AudiencesFrom(req.Context())
+	return &authenticator.Response{
+		User: &user.DefaultInfo{
+			Name:   "system:unsecured",
+			Groups: []string{user.SystemPrivilegedGroup, user.AllAuthenticated},
+		},
+		Audiences: auds,
+	}, true, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package server contains the plumbing to create kubernetes-like API server command.
+package server // import "k8s.io/apiserver/pkg/server"

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// certKeyContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}
+
+// sniCertKeyContent holds the content for the cert and key as well as any explicit names
+type sniCertKeyContent struct {
+	certKeyContent
+	sniNames []string
+}
+
+func (c *sniCertKeyContent) Equal(rhs *sniCertKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if len(c.sniNames) != len(rhs.sniNames) {
+		return false
+	}
+
+	for i := range c.sniNames {
+		if c.sniNames[i] != rhs.sniNames[i] {
+			return false
+		}
+	}
+
+	return c.certKeyContent.Equal(&rhs.certKeyContent)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// dynamicCertificateContent holds the content that overrides the baseTLSConfig
+type dynamicCertificateContent struct {
+	// clientCA holds the content for the clientCA bundle
+	clientCA    caBundleContent
+	servingCert certKeyContent
+	sniCerts    []sniCertKeyContent
+}
+
+// caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
+// method receiver.
+type caBundleContent struct {
+	caBundle []byte
+}
+
+func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if !c.clientCA.Equal(&rhs.clientCA) {
+		return false
+	}
+
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
+	if len(c.sniCerts) != len(rhs.sniCerts) {
+		return false
+	}
+
+	for i := range c.sniCerts {
+		if !c.sniCerts[i].Equal(&rhs.sniCerts[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.caBundle, rhs.caBundle)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes
+// It also fulfills the authenticator interface to provide verifyoptions
+type ConfigMapCAController struct {
+	name string
+
+	configmapLister    corev1listers.ConfigMapLister
+	configmapNamespace string
+	configmapName      string
+	configmapKey       string
+	// configMapInformer is tracked so that we can start these on Run
+	configMapInformer cache.SharedIndexInformer
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	queue workqueue.TypedRateLimitingInterface[string]
+	// preRunCaches are the caches to sync before starting the work of this control loop
+	preRunCaches []cache.InformerSynced
+}
+
+var _ CAContentProvider = &ConfigMapCAController{}
+var _ ControllerRunner = &ConfigMapCAController{}
+
+// NewDynamicCAFromConfigMapController returns a CAContentProvider based on a configmap that automatically reloads content.
+// It is near-realtime via an informer.
+func NewDynamicCAFromConfigMapController(purpose, namespace, name, key string, kubeClient kubernetes.Interface) (*ConfigMapCAController, error) {
+	if len(purpose) == 0 {
+		return nil, fmt.Errorf("missing purpose for ca bundle")
+	}
+	if len(namespace) == 0 {
+		return nil, fmt.Errorf("missing namespace for ca bundle")
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("missing name for ca bundle")
+	}
+	if len(key) == 0 {
+		return nil, fmt.Errorf("missing key for ca bundle")
+	}
+	caContentName := fmt.Sprintf("%s::%s::%s::%s", purpose, namespace, name, key)
+
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	uncastConfigmapInformer := corev1informers.NewFilteredConfigMapInformer(kubeClient, namespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *v1.ListOptions) {
+		listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String()
+	})
+
+	configmapLister := corev1listers.NewConfigMapLister(uncastConfigmapInformer.GetIndexer())
+
+	c := &ConfigMapCAController{
+		name:               caContentName,
+		configmapNamespace: namespace,
+		configmapName:      name,
+		configmapKey:       key,
+		configmapLister:    configmapLister,
+		configMapInformer:  uncastConfigmapInformer,
+
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)},
+		),
+		preRunCaches: []cache.InformerSynced{uncastConfigmapInformer.HasSynced},
+	}
+
+	uncastConfigmapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+		},
+	})
+
+	return c, nil
+}
+
+func (c *ConfigMapCAController) keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return c.configmapNamespace + "/" + c.configmapName
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *ConfigMapCAController) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *ConfigMapCAController) loadCABundle() error {
+	configMap, err := c.configmapLister.ConfigMaps(c.configmapNamespace).Get(c.configmapName)
+	if err != nil {
+		return err
+	}
+	caBundle := configMap.Data[c.configmapKey]
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged([]byte(caBundle)) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), []byte(caBundle))
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *ConfigMapCAController) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *ConfigMapCAController) RunOnce(ctx context.Context) error {
+	// Ignore the error when running once because when using a dynamically loaded ca file, because we think it's better to have nothing for
+	// a brief time than completely crash.  If crashing is necessary, higher order logic like a healthcheck and cause failures.
+	_ = c.loadCABundle()
+	return nil
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *ConfigMapCAController) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// we have a personal informer that is narrowly scoped, start it.
+	go c.configMapInformer.Run(ctx.Done())
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync(c.name, ctx.Done(), c.preRunCaches...) {
+		return
+	}
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *ConfigMapCAController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *ConfigMapCAController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *ConfigMapCAController) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *ConfigMapCAController) CurrentCABundleContent() []byte {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return nil // this can happen if we've been unable load data from the apiserver for some reason
+	}
+
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *ConfigMapCAController) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		// This can happen if we've been unable load data from the apiserver for some reason.
+		// In this case, we should not accept any connections on the basis of this ca bundle.
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -0,0 +1,294 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"k8s.io/client-go/util/cert"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce(ctx context.Context) error
+
+	// Run should be called a go .Run
+	Run(ctx context.Context, workers int)
+}
+
+// DynamicFileCAContent provides a CAContentProvider that can dynamically react to new file content
+// It also fulfills the authenticator interface to provide verifyoptions
+type DynamicFileCAContent struct {
+	name string
+
+	// filename is the name the file to read.
+	filename string
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.TypedRateLimitingInterface[string]
+}
+
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
+type caBundleAndVerifier struct {
+	caBundle      []byte
+	verifyOptions x509.VerifyOptions
+}
+
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
+func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+	name := fmt.Sprintf("%s::%s", purpose, filename)
+
+	ret := &DynamicFileCAContent{
+		name:     name,
+		filename: filename,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)},
+		),
+	}
+	if err := ret.loadCABundle(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *DynamicFileCAContent) loadCABundle() error {
+	caBundle, err := os.ReadFile(c.filename)
+	if err != nil {
+		return err
+	}
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged(caBundle) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle)
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+	klog.V(2).InfoS("Loaded a new CA Bundle and Verifier", "name", c.Name())
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce(ctx context.Context) error {
+	return c.loadCABundle()
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (c *DynamicFileCAContent) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start the loop that watches the CA file until stopCh is closed.
+	go wait.Until(func() {
+		if err := c.watchCAFile(ctx.Done()); err != nil {
+			klog.ErrorS(err, "Failed to watch CA file, will retry later")
+		}
+	}, time.Minute, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *DynamicFileCAContent) watchCAFile(stopCh <-chan struct{}) error {
+	// Trigger a check here to ensure the content will be checked periodically even if the following watch fails.
+	c.queue.Add(workItemKey)
+
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return fmt.Errorf("error creating fsnotify watcher: %v", err)
+	}
+	defer w.Close()
+
+	if err = w.Add(c.filename); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
+	}
+	// Trigger a check in case the file is updated before the watch starts.
+	c.queue.Add(workItemKey)
+
+	for {
+		select {
+		case e := <-w.Events:
+			if err := c.handleWatchEvent(e, w); err != nil {
+				return err
+			}
+		case err := <-w.Errors:
+			return fmt.Errorf("received fsnotify error: %v", err)
+		case <-stopCh:
+			return nil
+		}
+	}
+}
+
+// handleWatchEvent triggers reloading the CA file, and restarts a new watch if it's a Remove or Rename event.
+func (c *DynamicFileCAContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
+	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
+	defer c.queue.Add(workItemKey)
+	if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
+		return nil
+	}
+	if err := w.Remove(c.filename); err != nil && !errors.Is(err, fsnotify.ErrNonExistentWatch) {
+		klog.InfoS("Failed to remove file watch, it may have been deleted", "file", c.filename, "err", err)
+	}
+	if err := w.Add(c.filename); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
+	}
+	return nil
+}
+
+func (c *DynamicFileCAContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileCAContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}
+
+// newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.
+// It will return a nil function if you pass an empty CA file.
+func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) {
+	if len(caBundle) == 0 {
+		return nil, fmt.Errorf("missing content for CA bundle %q", name)
+	}
+
+	// Wrap with an x509 verifier
+	var err error
+	verifyOptions := defaultVerifyOptions()
+	verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle)
+	if err != nil {
+		return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err)
+	}
+
+	return &caBundleAndVerifier{
+		caBundle:      caBundle,
+		verifyOptions: verifyOptions,
+	}, nil
+}
+
+// defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time,
+// and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth)
+func defaultVerifyOptions() x509.VerifyOptions {
+	return x509.VerifyOptions{
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -0,0 +1,236 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"os"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// DynamicCertKeyPairContent provides a CertKeyContentProvider that can dynamically react to new file content
+type DynamicCertKeyPairContent struct {
+	name string
+
+	// certFile is the name of the certificate file to read.
+	certFile string
+	// keyFile is the name of the key file to read.
+	keyFile string
+
+	// certKeyPair is a certKeyContent that contains the last read, non-zero length content of the key and cert
+	certKeyPair atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.TypedRateLimitingInterface[string]
+}
+
+var _ CertKeyContentProvider = &DynamicCertKeyPairContent{}
+var _ ControllerRunner = &DynamicCertKeyPairContent{}
+
+// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename
+func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicCertKeyPairContent, error) {
+	if len(certFile) == 0 || len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for serving cert")
+	}
+	name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile)
+
+	ret := &DynamicCertKeyPairContent{
+		name:     name,
+		certFile: certFile,
+		keyFile:  keyFile,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)},
+		),
+	}
+	if err := ret.loadCertKeyPair(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the serving cert content changes.
+func (c *DynamicCertKeyPairContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCertKeyPair determines the next set of content for the file.
+func (c *DynamicCertKeyPairContent) loadCertKeyPair() error {
+	cert, err := os.ReadFile(c.certFile)
+	if err != nil {
+		return err
+	}
+	key, err := os.ReadFile(c.keyFile)
+	if err != nil {
+		return err
+	}
+	if len(cert) == 0 || len(key) == 0 {
+		return fmt.Errorf("missing content for serving cert %q", c.Name())
+	}
+
+	// Ensure that the key matches the cert and both are valid
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return err
+	}
+
+	newCertKey := &certKeyContent{
+		cert: cert,
+		key:  key,
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := c.certKeyPair.Load().(*certKeyContent)
+	if ok && existing != nil && existing.Equal(newCertKey) {
+		return nil
+	}
+
+	c.certKeyPair.Store(newCertKey)
+	klog.V(2).InfoS("Loaded a new cert/key pair", "name", c.Name())
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicCertKeyPairContent) RunOnce(ctx context.Context) error {
+	return c.loadCertKeyPair()
+}
+
+// Run starts the controller and blocks until context is killed.
+func (c *DynamicCertKeyPairContent) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start the loop that watches the cert and key files until stopCh is closed.
+	go wait.Until(func() {
+		if err := c.watchCertKeyFile(ctx.Done()); err != nil {
+			klog.ErrorS(err, "Failed to watch cert and key file, will retry later")
+		}
+	}, time.Minute, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *DynamicCertKeyPairContent) watchCertKeyFile(stopCh <-chan struct{}) error {
+	// Trigger a check here to ensure the content will be checked periodically even if the following watch fails.
+	c.queue.Add(workItemKey)
+
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return fmt.Errorf("error creating fsnotify watcher: %v", err)
+	}
+	defer w.Close()
+
+	if err := w.Add(c.certFile); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.certFile, err)
+	}
+	if err := w.Add(c.keyFile); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.keyFile, err)
+	}
+	// Trigger a check in case the file is updated before the watch starts.
+	c.queue.Add(workItemKey)
+
+	for {
+		select {
+		case e := <-w.Events:
+			if err := c.handleWatchEvent(e, w); err != nil {
+				return err
+			}
+		case err := <-w.Errors:
+			return fmt.Errorf("received fsnotify error: %v", err)
+		case <-stopCh:
+			return nil
+		}
+	}
+}
+
+// handleWatchEvent triggers reloading the cert and key file, and restarts a new watch if it's a Remove or Rename event.
+// If one file is updated before the other, the loadCertKeyPair method will catch the mismatch and will not apply the
+// change. When an event of the other file is received, it will trigger reloading the files again and the new content
+// will be loaded and used.
+func (c *DynamicCertKeyPairContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
+	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
+	defer c.queue.Add(workItemKey)
+	if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
+		return nil
+	}
+	if err := w.Remove(e.Name); err != nil {
+		klog.InfoS("Failed to remove file watch, it may have been deleted", "file", e.Name, "err", err)
+	}
+	if err := w.Add(e.Name); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", e.Name, err)
+	}
+	return nil
+}
+
+func (c *DynamicCertKeyPairContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicCertKeyPairContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCertKeyPair()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicCertKeyPairContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key byte content
+func (c *DynamicCertKeyPairContent) CurrentCertKeyContent() ([]byte, []byte) {
+	certKeyContent := c.certKeyPair.Load().(*certKeyContent)
+	return certKeyContent.cert, certKeyContent.key
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content
+type DynamicFileSNIContent struct {
+	*DynamicCertKeyPairContent
+	sniNames []string
+}
+
+var _ SNICertKeyContentProvider = &DynamicFileSNIContent{}
+var _ ControllerRunner = &DynamicFileSNIContent{}
+
+// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names
+func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) {
+	servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &DynamicFileSNIContent{
+		DynamicCertKeyPairContent: servingContent,
+		sniNames:                  sniNames,
+	}
+	if err := ret.loadCertKeyPair(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// SNINames returns explicitly set SNI names for the certificate. These are not dynamic.
+func (c *DynamicFileSNIContent) SNINames() []string {
+	return c.sniNames
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/interfaces.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+)
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
+	Enqueue()
+}
+
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes.
+	// This is a noop on static providers.
+	AddListener(listener Listener)
+}
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	Notifier
+
+	// Name is just an identifier.
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content. Errors can be
+	// contained to the controllers initializing the value. By the time you get
+	// here, you should always be returning a value that won't fail.
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators.
+	VerifyOptions() (x509.VerifyOptions, bool)
+}
+
+// CertKeyContentProvider provides a certificate and matching private key.
+type CertKeyContentProvider interface {
+	Notifier
+
+	// Name is just an identifier.
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content.
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// SNICertKeyContentProvider provides a certificate and matching private key as
+// well as optional explicit names.
+type SNICertKeyContentProvider interface {
+	Notifier
+
+	CertKeyContentProvider
+	// SNINames provides names used for SNI. May return nil.
+	SNINames() []string
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/named_certificates.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+)
+
+// BuildNamedCertificates returns a map of *tls.Certificate by name. It's
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// is invalid. Returns nil if len(certs) == 0
+func (c *DynamicServingCertificateController) BuildNamedCertificates(sniCerts []sniCertKeyContent) (map[string]*tls.Certificate, error) {
+	nameToCertificate := map[string]*tls.Certificate{}
+	byNameExplicit := map[string]*tls.Certificate{}
+
+	// Iterate backwards so that earlier certs take precedence in the names map
+	for i := len(sniCerts) - 1; i >= 0; i-- {
+		cert, err := tls.X509KeyPair(sniCerts[i].cert, sniCerts[i].key)
+		if err != nil {
+			return nil, fmt.Errorf("invalid SNI cert keypair [%d/%q]: %v", i, c.sniCerts[i].Name(), err)
+		}
+
+		// error is not possible given above call to X509KeyPair
+		x509Cert, _ := x509.ParseCertificate(cert.Certificate[0])
+
+		names := sniCerts[i].sniNames
+		for _, name := range names {
+			byNameExplicit[name] = &cert
+		}
+
+		klog.V(2).InfoS("Loaded SNI cert", "index", i, "certName", c.sniCerts[i].Name(), "certDetail", GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.sniCerts[i].Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "SNICertificateReload", "loaded SNI cert [%d/%q]: %s with explicit names %v", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert), names)
+		}
+
+		if len(names) == 0 {
+			names = getCertificateNames(x509Cert)
+			for _, name := range names {
+				nameToCertificate[name] = &cert
+			}
+		}
+	}
+
+	// Explicitly set names must override
+	for k, v := range byNameExplicit {
+		nameToCertificate[k] = v
+	}
+
+	return nameToCertificate, nil
+}
+
+// getCertificateNames returns names for an x509.Certificate. The names are
+// suitable for use in tls.Config#NamedCertificates.
+func getCertificateNames(cert *x509.Certificate) []string {
+	var names []string
+
+	cn := cert.Subject.CommonName
+	cnIsIP := netutils.ParseIPSloppy(cn) != nil
+	cnIsValidDomain := cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0
+	// don't use the CN if it is a valid IP because our IP serving detection may unexpectedly use it to terminate the connection.
+	if !cnIsIP && cnIsValidDomain {
+		names = append(names, cn)
+	}
+	names = append(names, cert.DNSNames...)
+	// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
+	// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
+
+	return names
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+)
+
+type staticCAContent struct {
+	name     string
+	caBundle *caBundleAndVerifier
+}
+
+var _ CAContentProvider = &staticCAContent{}
+
+// NewStaticCAContent returns a CAContentProvider that always returns the same value
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCAContent{
+		name:     name,
+		caBundle: caBundleAndVerifier,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCAContent) Name() string {
+	return c.name
+}
+
+func (c *staticCAContent) AddListener(Listener) {}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	return c.caBundle.verifyOptions, true
+}
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+func (c *staticCertKeyContent) AddListener(Listener) {}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}
+
+type staticSNICertKeyContent struct {
+	staticCertKeyContent
+	sniNames []string
+}
+
+// NewStaticSNICertKeyContent returns a SNICertKeyContentProvider that always returns the same value
+func NewStaticSNICertKeyContent(name string, cert, key []byte, sniNames ...string) (SNICertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticSNICertKeyContent{
+		staticCertKeyContent: staticCertKeyContent{
+			name: name,
+			cert: cert,
+			key:  key,
+		},
+		sniNames: sniNames,
+	}, nil
+}
+
+func (c *staticSNICertKeyContent) SNINames() []string {
+	return c.sniNames
+}
+
+func (c *staticSNICertKeyContent) AddListener(Listener) {}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -0,0 +1,287 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+const workItemKey = "key"
+
+// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
+type DynamicServingCertificateController struct {
+	// baseTLSConfig is the static portion of the tlsConfig for serving to clients.  It is copied and the copy is mutated
+	// based on the dynamic cert state.
+	baseTLSConfig *tls.Config
+
+	// clientCA provides the very latest content of the ca bundle
+	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
+	// sniCerts are a list of CertKeyContentProvider with associated names used for SNI
+	sniCerts []SNICertKeyContentProvider
+
+	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
+	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
+	currentlyServedContent *dynamicCertificateContent
+	// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
+	currentServingTLSConfig atomic.Value
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue         workqueue.TypedRateLimitingInterface[string]
+	eventRecorder events.EventRecorder
+}
+
+var _ Listener = &DynamicServingCertificateController{}
+
+// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
+func NewDynamicServingCertificateController(
+	baseTLSConfig *tls.Config,
+	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
+	sniCerts []SNICertKeyContentProvider,
+	eventRecorder events.EventRecorder,
+) *DynamicServingCertificateController {
+	c := &DynamicServingCertificateController{
+		baseTLSConfig: baseTLSConfig,
+		clientCA:      clientCA,
+		servingCert:   servingCert,
+		sniCerts:      sniCerts,
+
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: "DynamicServingCertificateController"},
+		),
+		eventRecorder: eventRecorder,
+	}
+
+	return c
+}
+
+// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
+func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+	uncastObj := c.currentServingTLSConfig.Load()
+	if uncastObj == nil {
+		return nil, errors.New("dynamiccertificates: configuration not ready")
+	}
+	tlsConfig, ok := uncastObj.(*tls.Config)
+	if !ok {
+		return nil, errors.New("dynamiccertificates: unexpected config type")
+	}
+
+	tlsConfigCopy := tlsConfig.Clone()
+
+	// if the client set SNI information, just use our "normal" SNI flow
+	if len(clientHello.ServerName) > 0 {
+		return tlsConfigCopy, nil
+	}
+
+	// if the client didn't set SNI, then we need to inspect the requested IP so that we can choose
+	// a certificate from our list if we specifically handle that IP.  This can happen when an IP is specifically mapped by name.
+	host, _, err := net.SplitHostPort(clientHello.Conn.LocalAddr().String())
+	if err != nil {
+		return tlsConfigCopy, nil
+	}
+
+	ipCert, ok := tlsConfigCopy.NameToCertificate[host]
+	if !ok {
+		return tlsConfigCopy, nil
+	}
+	tlsConfigCopy.Certificates = []tls.Certificate{*ipCert}
+	tlsConfigCopy.NameToCertificate = nil
+
+	return tlsConfigCopy, nil
+}
+
+// newTLSContent determines the next set of content for overriding the baseTLSConfig.
+func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
+	newContent := &dynamicCertificateContent{}
+
+	if c.clientCA != nil {
+		currClientCABundle := c.clientCA.CurrentCABundleContent()
+		// we allow removing all client ca bundles because the server is still secure when this happens. it just means
+		// that there isn't a hint to clients about which client-cert to used.  this happens when there is no client-ca
+		// yet known for authentication, which can happen in aggregated apiservers and some kube-apiserver deployment modes.
+		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
+	}
+
+	for i, sniCert := range c.sniCerts {
+		currCert, currKey := sniCert.CurrentCertKeyContent()
+		if len(currCert) == 0 || len(currKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty SNI certificate from %d/%q", i, sniCert.Name())
+		}
+
+		newContent.sniCerts = append(newContent.sniCerts, sniCertKeyContent{certKeyContent: certKeyContent{cert: currCert, key: currKey}, sniNames: sniCert.SNINames()})
+	}
+
+	return newContent, nil
+}
+
+// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
+// GetConfigForClient.
+func (c *DynamicServingCertificateController) syncCerts() error {
+	newContent, err := c.newTLSContent()
+	if err != nil {
+		return err
+	}
+	// if the content is the same as what we currently have, we can simply skip it.  This works because we are single
+	// threaded.  If you ever make this multi-threaded, add a lock.
+	if newContent.Equal(c.currentlyServedContent) {
+		return nil
+	}
+
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
+	// parse new content to add to TLSConfig
+	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
+		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file %q: %v", string(newContent.clientCA.caBundle), err)
+		}
+		for i, cert := range newClientCAs {
+			klog.V(2).InfoS("Loaded client CA", "index", i, "certName", c.clientCA.Name(), "certDetail", GetHumanCertDetail(cert))
+			if c.eventRecorder != nil {
+				c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.clientCA.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			}
+
+			newClientCAPool.AddCert(cert)
+		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
+	}
+
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).InfoS("Loaded serving cert", "certName", c.servingCert.Name(), "certDetail", GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.servingCert.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+	}
+
+	if len(newContent.sniCerts) > 0 {
+		newTLSConfigCopy.NameToCertificate, err = c.BuildNamedCertificates(newContent.sniCerts)
+		if err != nil {
+			return fmt.Errorf("unable to build named certificate map: %v", err)
+		}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if servingCert is not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, sniCert := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *sniCert)
+		}
+	}
+
+	// store new values of content for serving.
+	c.currentServingTLSConfig.Store(newTLSConfigCopy)
+	c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
+
+	return nil
+}
+
+// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
+func (c *DynamicServingCertificateController) RunOnce() error {
+	return c.syncCerts()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting DynamicServingCertificateController")
+	defer klog.InfoS("Shutting down DynamicServingCertificateController")
+
+	// synchronously load once.  We will trigger again, so ignoring any error is fine
+	_ = c.RunOnce()
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.Until(func() {
+		c.Enqueue()
+	}, 1*time.Minute, stopCh)
+
+	<-stopCh
+}
+
+func (c *DynamicServingCertificateController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicServingCertificateController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.syncCerts()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
+func (c *DynamicServingCertificateController) Enqueue() {
+	c.queue.Add(workItemKey)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+type unionCAContent []CAContentProvider
+
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
+// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
+func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
+	return unionCAContent(caContentProviders)
+}
+
+// Name is just an identifier
+func (c unionCAContent) Name() string {
+	names := []string{}
+	for _, curr := range c {
+		names = append(names, curr.Name())
+	}
+	return strings.Join(names, ",")
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) CurrentCABundleContent() []byte {
+	caBundles := [][]byte{}
+	for _, curr := range c {
+		if currCABytes := curr.CurrentCABundleContent(); len(currCABytes) > 0 {
+			caBundles = append(caBundles, []byte(strings.TrimSpace(string(currCABytes))))
+		}
+	}
+
+	return bytes.Join(caBundles, []byte("\n"))
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	currCABundle := c.CurrentCABundleContent()
+	if len(currCABundle) == 0 {
+		return x509.VerifyOptions{}, false
+	}
+
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions, true
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		curr.AddListener(listener)
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce(ctx context.Context) error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(ctx); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(ctx context.Context, workers int) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(ctx, workers)
+		}
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
+// kube-apiserver usage of certs.
+func GetHumanCertDetail(certificate *x509.Certificate) string {
+	humanName := certificate.Subject.CommonName
+	signerHumanName := certificate.Issuer.CommonName
+	if certificate.Subject.CommonName == certificate.Issuer.CommonName {
+		signerHumanName = "<self>"
+	}
+
+	usages := []string{}
+	for _, curr := range certificate.ExtKeyUsage {
+		if curr == x509.ExtKeyUsageClientAuth {
+			usages = append(usages, "client")
+			continue
+		}
+		if curr == x509.ExtKeyUsageServerAuth {
+			usages = append(usages, "serving")
+			continue
+		}
+
+		usages = append(usages, fmt.Sprintf("%d", curr))
+	}
+
+	validServingNames := []string{}
+	for _, ip := range certificate.IPAddresses {
+		validServingNames = append(validServingNames, ip.String())
+	}
+	validServingNames = append(validServingNames, certificate.DNSNames...)
+	servingString := ""
+	if len(validServingNames) > 0 {
+		servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
+	}
+
+	groupString := ""
+	if len(certificate.Subject.Organization) > 0 {
+		groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
+	}
+
+	return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
+		time.Now().UTC())
+}

e2e/vendor/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -0,0 +1,247 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/utils/path"
+)
+
+var cfgScheme = runtime.NewScheme()
+
+// validEgressSelectorNames contains the set of valid egress selctor names.
+var validEgressSelectorNames = sets.NewString("controlplane", "cluster", "etcd")
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path.
+// It returns the loaded egress selector configuration if the input file aligns with the required syntax.
+// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op.
+// It does this by returning a nil configuration, which preserves backward compatibility.
+// This works because prior to this there was no egress selector configuration.
+// It returns an error if the file did not exist.
+func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) {
+	if configFilePath == "" {
+		return nil, nil
+	}
+	// a file was provided, so we just read it.
+	data, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
+	}
+	config, gvk, err := serializer.NewCodecFactory(cfgScheme, serializer.EnableStrict).UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	internalConfig, ok := config.(*apiserver.EgressSelectorConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("unexpected config type: %v", gvk)
+	}
+	return internalConfig, nil
+}
+
+// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for
+// common configuration errors. It will return error for problems such as configuring mtls/cert
+// settings for protocol which do not support security. It will also try to catch errors such as
+// incorrect file paths. It will return nil if it does not find anything wrong.
+func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if config == nil {
+		return allErrs // Treating a nil configuration as valid
+	}
+	for _, service := range config.EgressSelections {
+		fldPath := field.NewPath("service", "connection")
+		switch service.Connection.ProxyProtocol {
+		case apiserver.ProtocolDirect:
+			allErrs = append(allErrs, validateDirectConnection(service.Connection, fldPath)...)
+		case apiserver.ProtocolHTTPConnect:
+			allErrs = append(allErrs, validateHTTPConnectTransport(service.Connection.Transport, fldPath)...)
+		case apiserver.ProtocolGRPC:
+			allErrs = append(allErrs, validateGRPCTransport(service.Connection.Transport, fldPath)...)
+		default:
+			allErrs = append(allErrs, field.NotSupported(
+				fldPath.Child("protocol"),
+				service.Connection.ProxyProtocol,
+				[]string{
+					string(apiserver.ProtocolDirect),
+					string(apiserver.ProtocolHTTPConnect),
+					string(apiserver.ProtocolGRPC),
+				}))
+		}
+	}
+
+	seen := sets.String{}
+	for i, service := range config.EgressSelections {
+		canonicalName := strings.ToLower(service.Name)
+		fldPath := field.NewPath("service", "connection")
+		// no duplicate check
+		if seen.Has(canonicalName) {
+			allErrs = append(allErrs, field.Duplicate(fldPath.Index(i), canonicalName))
+			continue
+		}
+		seen.Insert(canonicalName)
+
+		if !validEgressSelectorNames.Has(canonicalName) {
+			allErrs = append(allErrs, field.NotSupported(fldPath, canonicalName, validEgressSelectorNames.List()))
+			continue
+		}
+	}
+
+	return allErrs
+}
+
+func validateHTTPConnectTransport(transport *apiserver.Transport, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if transport == nil {
+		allErrs = append(allErrs, field.Required(
+			fldPath.Child("transport"),
+			"transport must be set for HTTPConnect"))
+		return allErrs
+	}
+
+	if transport.TCP != nil && transport.UDS != nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("tcp"),
+			transport.TCP,
+			"TCP and UDS cannot both be set"))
+	} else if transport.TCP == nil && transport.UDS == nil {
+		allErrs = append(allErrs, field.Required(
+			fldPath.Child("tcp"),
+			"One of TCP or UDS must be set"))
+	} else if transport.TCP != nil {
+		allErrs = append(allErrs, validateTCPConnection(transport.TCP, fldPath)...)
+	} else if transport.UDS != nil {
+		allErrs = append(allErrs, validateUDSConnection(transport.UDS, fldPath)...)
+	}
+	return allErrs
+}
+
+func validateGRPCTransport(transport *apiserver.Transport, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if transport == nil {
+		allErrs = append(allErrs, field.Required(
+			fldPath.Child("transport"),
+			"transport must be set for GRPC"))
+		return allErrs
+	}
+
+	if transport.UDS != nil {
+		allErrs = append(allErrs, validateUDSConnection(transport.UDS, fldPath)...)
+	} else {
+		allErrs = append(allErrs, field.Required(
+			fldPath.Child("uds"),
+			"UDS must be set with GRPC"))
+	}
+	return allErrs
+}
+
+func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	if connection.Transport != nil {
+		return field.ErrorList{field.Invalid(
+			fldPath.Child("transport"),
+			"direct",
+			"Transport config should be absent for direct connect"),
+		}
+	}
+
+	return nil
+}
+
+func validateUDSConnection(udsConfig *apiserver.UDSTransport, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if udsConfig.UDSName == "" {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("udsName"),
+			"nil",
+			"UDSName should be present for UDS connections"))
+	}
+	return allErrs
+}
+
+func validateTCPConnection(tcpConfig *apiserver.TCPTransport, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if strings.HasPrefix(tcpConfig.URL, "http://") {
+		if tcpConfig.TLSConfig != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("tlsConfig"),
+				"nil",
+				"TLSConfig config should not be present when using HTTP"))
+		}
+	} else if strings.HasPrefix(tcpConfig.URL, "https://") {
+		return validateTLSConfig(tcpConfig.TLSConfig, fldPath)
+	} else {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("url"),
+			tcpConfig.URL,
+			"supported connection protocols are http:// and https://"))
+	}
+	return allErrs
+}
+
+func validateTLSConfig(tlsConfig *apiserver.TLSConfig, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if tlsConfig == nil {
+		allErrs = append(allErrs, field.Required(
+			fldPath.Child("tlsConfig"),
+			"TLSConfig must be present when using HTTPS"))
+		return allErrs
+	}
+	if tlsConfig.CABundle != "" {
+		if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.CABundle); !exists || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("tlsConfig", "caBundle"),
+				tlsConfig.CABundle,
+				"TLS config ca bundle does not exist"))
+		}
+	}
+	if tlsConfig.ClientCert == "" {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("tlsConfig", "clientCert"),
+			"nil",
+			"Using TLS requires clientCert"))
+	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientCert); !exists || err != nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("tlsConfig", "clientCert"),
+			tlsConfig.ClientCert,
+			"TLS client cert does not exist"))
+	}
+	if tlsConfig.ClientKey == "" {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("tlsConfig", "clientKey"),
+			"nil",
+			"Using TLS requires requires clientKey"))
+	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientKey); !exists || err != nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("tlsConfig", "clientKey"),
+			tlsConfig.ClientKey,
+			"TLS client key does not exist"))
+	}
+	return allErrs
+}

e2e/vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -0,0 +1,406 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"go.opentelemetry.io/otel/attribute"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/component-base/tracing"
+	"k8s.io/klog/v2"
+	client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
+)
+
+var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
+
+func init() {
+	client.Metrics.RegisterMetrics(legacyregistry.Registerer())
+}
+
+// EgressSelector is the map of network context type to context dialer, for network egress.
+type EgressSelector struct {
+	egressToDialer map[EgressType]utilnet.DialFunc
+}
+
+// EgressType is an indicator of which egress selection should be used for sending traffic.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md#network-context
+type EgressType int
+
+const (
+	// ControlPlane is the EgressType for traffic intended to go to the control plane.
+	ControlPlane EgressType = iota
+	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
+	Etcd
+	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
+	Cluster
+)
+
+// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
+type NetworkContext struct {
+	// EgressSelectionName is the unique name of the
+	// EgressSelectorConfiguration which determines
+	// the network we route the traffic to.
+	EgressSelectionName EgressType
+}
+
+// Lookup is the interface to get the dialer function for the network context.
+type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
+
+// String returns the canonical string representation of the egress type
+func (s EgressType) String() string {
+	switch s {
+	case ControlPlane:
+		return "controlplane"
+	case Etcd:
+		return "etcd"
+	case Cluster:
+		return "cluster"
+	default:
+		return "invalid"
+	}
+}
+
+// AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
+func (s EgressType) AsNetworkContext() NetworkContext {
+	return NetworkContext{EgressSelectionName: s}
+}
+
+func lookupServiceName(name string) (EgressType, error) {
+	switch strings.ToLower(name) {
+	case "controlplane":
+		return ControlPlane, nil
+	case "etcd":
+		return Etcd, nil
+	case "cluster":
+		return Cluster, nil
+	}
+	return -1, fmt.Errorf("unrecognized service name %s", name)
+}
+
+func tunnelHTTPConnect(proxyConn net.Conn, proxyAddress, addr string) (net.Conn, error) {
+	fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
+	br := bufio.NewReader(proxyConn)
+	res, err := http.ReadResponse(br, nil)
+	if err != nil {
+		proxyConn.Close()
+		return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
+			addr, proxyAddress, err)
+	}
+	if res.StatusCode != 200 {
+		proxyConn.Close()
+		return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
+			proxyAddress, addr, res.StatusCode, res.Status)
+	}
+
+	// It's safe to discard the bufio.Reader here and return the
+	// original TCP conn directly because we only use this for
+	// TLS, and in TLS the client speaks first, so we know there's
+	// no unbuffered data. But we can double-check.
+	if br.Buffered() > 0 {
+		proxyConn.Close()
+		return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
+			br.Buffered(), proxyAddress)
+	}
+	return proxyConn, nil
+}
+
+type proxier interface {
+	// proxy returns a connection to addr.
+	proxy(ctx context.Context, addr string) (net.Conn, error)
+}
+
+var _ proxier = &httpConnectProxier{}
+
+type httpConnectProxier struct {
+	conn         net.Conn
+	proxyAddress string
+}
+
+func (t *httpConnectProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
+	return tunnelHTTPConnect(t.conn, t.proxyAddress, addr)
+}
+
+var _ proxier = &grpcProxier{}
+
+type grpcProxier struct {
+	tunnel client.Tunnel
+}
+
+func (g *grpcProxier) proxy(ctx context.Context, addr string) (net.Conn, error) {
+	return g.tunnel.DialContext(ctx, "tcp", addr)
+}
+
+type proxyServerConnector interface {
+	// connect establishes connection to the proxy server, and returns a
+	// proxier based on the connection.
+	//
+	// The provided Context must be non-nil. The context is used for connecting to the proxy only.
+	// If the context expires before the connection is complete, an error is returned.
+	// Once successfully connected to the proxy, any expiration of the context will not affect the connection.
+	connect(context.Context) (proxier, error)
+}
+
+type tcpHTTPConnectConnector struct {
+	proxyAddress string
+	tlsConfig    *tls.Config
+}
+
+func (t *tcpHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
+	d := tls.Dialer{
+		Config: t.tlsConfig,
+	}
+	conn, err := d.DialContext(ctx, "tcp", t.proxyAddress)
+	if err != nil {
+		return nil, err
+	}
+	return &httpConnectProxier{conn: conn, proxyAddress: t.proxyAddress}, nil
+}
+
+type udsHTTPConnectConnector struct {
+	udsName string
+}
+
+func (u *udsHTTPConnectConnector) connect(ctx context.Context) (proxier, error) {
+	var d net.Dialer
+	conn, err := d.DialContext(ctx, "unix", u.udsName)
+	if err != nil {
+		return nil, err
+	}
+	return &httpConnectProxier{conn: conn, proxyAddress: u.udsName}, nil
+}
+
+type udsGRPCConnector struct {
+	udsName string
+}
+
+// connect establishes a connection to a proxy over gRPC.
+// TODO At the moment, it does not use the provided context.
+func (u *udsGRPCConnector) connect(_ context.Context) (proxier, error) {
+	udsName := u.udsName
+	dialOption := grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
+		var d net.Dialer
+		c, err := d.DialContext(ctx, "unix", udsName)
+		if err != nil {
+			klog.Errorf("failed to create connection to uds name %s, error: %v", udsName, err)
+		}
+		return c, err
+	})
+
+	// CreateSingleUseGrpcTunnel() unfortunately couples dial and connection contexts. Because of that,
+	// we cannot use ctx just for dialing and control the connection lifetime separately.
+	// See https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/357.
+	tunnelCtx := context.TODO()
+	tunnel, err := client.CreateSingleUseGrpcTunnel(tunnelCtx, udsName, dialOption,
+		grpc.WithBlock(),
+		grpc.WithReturnConnectionError(),
+		grpc.WithTimeout(30*time.Second), // matches http.DefaultTransport dial timeout
+		grpc.WithTransportCredentials(insecure.NewCredentials()))
+	if err != nil {
+		return nil, err
+	}
+	return &grpcProxier{tunnel: tunnel}, nil
+}
+
+type dialerCreator struct {
+	connector proxyServerConnector
+	direct    bool
+	options   metricsOptions
+}
+
+type metricsOptions struct {
+	transport string
+	protocol  string
+}
+
+func (d *dialerCreator) createDialer() utilnet.DialFunc {
+	if d.direct {
+		return directDialer
+	}
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		ctx, span := tracing.Start(ctx, fmt.Sprintf("Proxy via %s protocol over %s", d.options.protocol, d.options.transport), attribute.String("address", addr))
+		defer span.End(500 * time.Millisecond)
+		start := egressmetrics.Metrics.Clock().Now()
+		egressmetrics.Metrics.ObserveDialStart(d.options.protocol, d.options.transport)
+		proxier, err := d.connector.connect(ctx)
+		if err != nil {
+			egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageConnect)
+			return nil, err
+		}
+		conn, err := proxier.proxy(ctx, addr)
+		if err != nil {
+			egressmetrics.Metrics.ObserveDialFailure(d.options.protocol, d.options.transport, egressmetrics.StageProxy)
+			return nil, err
+		}
+		egressmetrics.Metrics.ObserveDialLatency(egressmetrics.Metrics.Clock().Now().Sub(start), d.options.protocol, d.options.transport)
+		return conn, nil
+	}
+}
+
+func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) {
+	clientCert := t.ClientCert
+	clientKey := t.ClientKey
+	caCert := t.CABundle
+	clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
+	}
+	certPool := x509.NewCertPool()
+	if caCert != "" {
+		certBytes, err := os.ReadFile(caCert)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
+		}
+		ok := certPool.AppendCertsFromPEM(certBytes)
+		if !ok {
+			return nil, fmt.Errorf("failed to append CA cert to the cert pool")
+		}
+	} else {
+		// Use host's root CA set instead of providing our own
+		certPool = nil
+	}
+	return &tls.Config{
+		Certificates: []tls.Certificate{clientCerts},
+		RootCAs:      certPool,
+	}, nil
+}
+
+func getProxyAddress(urlString string) (string, error) {
+	proxyURL, err := url.Parse(urlString)
+	if err != nil {
+		return "", fmt.Errorf("invalid proxy server url %q: %v", urlString, err)
+	}
+	return proxyURL.Host, nil
+}
+
+func connectionToDialerCreator(c apiserver.Connection) (*dialerCreator, error) {
+	switch c.ProxyProtocol {
+
+	case apiserver.ProtocolHTTPConnect:
+		if c.Transport.UDS != nil {
+			return &dialerCreator{
+				connector: &udsHTTPConnectConnector{
+					udsName: c.Transport.UDS.UDSName,
+				},
+				options: metricsOptions{
+					transport: egressmetrics.TransportUDS,
+					protocol:  egressmetrics.ProtocolHTTPConnect,
+				},
+			}, nil
+		} else if c.Transport.TCP != nil {
+			tlsConfig, err := getTLSConfig(c.Transport.TCP.TLSConfig)
+			if err != nil {
+				return nil, err
+			}
+			proxyAddress, err := getProxyAddress(c.Transport.TCP.URL)
+			if err != nil {
+				return nil, err
+			}
+			return &dialerCreator{
+				connector: &tcpHTTPConnectConnector{
+					tlsConfig:    tlsConfig,
+					proxyAddress: proxyAddress,
+				},
+				options: metricsOptions{
+					transport: egressmetrics.TransportTCP,
+					protocol:  egressmetrics.ProtocolHTTPConnect,
+				},
+			}, nil
+		} else {
+			return nil, fmt.Errorf("Either a TCP or UDS transport must be specified")
+		}
+	case apiserver.ProtocolGRPC:
+		if c.Transport.UDS != nil {
+			return &dialerCreator{
+				connector: &udsGRPCConnector{
+					udsName: c.Transport.UDS.UDSName,
+				},
+				options: metricsOptions{
+					transport: egressmetrics.TransportUDS,
+					protocol:  egressmetrics.ProtocolGRPC,
+				},
+			}, nil
+		}
+		return nil, fmt.Errorf("UDS transport must be specified for GRPC")
+	case apiserver.ProtocolDirect:
+		return &dialerCreator{direct: true}, nil
+	default:
+		return nil, fmt.Errorf("unrecognized service connection protocol %q", c.ProxyProtocol)
+	}
+
+}
+
+// NewEgressSelector configures lookup mechanism for Lookup.
+// It does so based on a EgressSelectorConfiguration which was read at startup.
+func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
+	if config == nil || config.EgressSelections == nil {
+		// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
+		return nil, nil
+	}
+	cs := &EgressSelector{
+		egressToDialer: make(map[EgressType]utilnet.DialFunc),
+	}
+	for _, service := range config.EgressSelections {
+		name, err := lookupServiceName(service.Name)
+		if err != nil {
+			return nil, err
+		}
+		dialerCreator, err := connectionToDialerCreator(service.Connection)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create dialer for egressSelection %q: %v", name, err)
+		}
+		cs.egressToDialer[name] = dialerCreator.createDialer()
+	}
+	return cs, nil
+}
+
+// NewEgressSelectorWithMap returns a EgressSelector with the supplied EgressType to DialFunc map.
+func NewEgressSelectorWithMap(m map[EgressType]utilnet.DialFunc) *EgressSelector {
+	if m == nil {
+		m = make(map[EgressType]utilnet.DialFunc)
+	}
+	return &EgressSelector{
+		egressToDialer: m,
+	}
+}
+
+// Lookup gets the dialer function for the network context.
+// This is configured for the Kubernetes API Server at startup.
+func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
+	if cs.egressToDialer == nil {
+		// The round trip wrapper will over-ride the dialContext method appropriately
+		return nil, nil
+	}
+
+	return cs.egressToDialer[networkContext.EgressSelectionName], nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/egressselector/metrics/metrics.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/utils/clock"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "egress_dialer"
+
+	// ProtocolHTTPConnect means that the proxy protocol is http-connect.
+	ProtocolHTTPConnect = "http_connect"
+	// ProtocolGRPC means that the proxy protocol is the GRPC protocol.
+	ProtocolGRPC = "grpc"
+	// TransportTCP means that the transport is TCP.
+	TransportTCP = "tcp"
+	// TransportUDS means that the transport is UDS.
+	TransportUDS = "uds"
+	// StageConnect indicates that the dial failed at establishing connection to the proxy server.
+	StageConnect = "connect"
+	// StageProxy indicates that the dial failed at requesting the proxy server to proxy.
+	StageProxy = "proxy"
+)
+
+var (
+	// Use buckets ranging from 5 ms to 12.5 seconds.
+	latencyBuckets = []float64{0.005, 0.025, 0.1, 0.5, 2.5, 12.5}
+
+	// Metrics provides access to all dial metrics.
+	Metrics = newDialMetrics()
+)
+
+// DialMetrics instruments dials to proxy server with prometheus metrics.
+type DialMetrics struct {
+	clock     clock.Clock
+	starts    *metrics.CounterVec
+	latencies *metrics.HistogramVec
+	failures  *metrics.CounterVec
+}
+
+// newDialMetrics create a new DialMetrics, configured with default metric names.
+func newDialMetrics() *DialMetrics {
+	starts := metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dial_start_total",
+			Help:           "Dial starts, labeled by the protocol (http-connect or grpc) and transport (tcp or uds).",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"protocol", "transport"},
+	)
+
+	latencies := metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dial_duration_seconds",
+			Help:           "Dial latency histogram in seconds, labeled by the protocol (http-connect or grpc), transport (tcp or uds)",
+			Buckets:        latencyBuckets,
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"protocol", "transport"},
+	)
+
+	failures := metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dial_failure_count",
+			Help:           "Dial failure count, labeled by the protocol (http-connect or grpc), transport (tcp or uds), and stage (connect or proxy). The stage indicates at which stage the dial failed",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"protocol", "transport", "stage"},
+	)
+
+	legacyregistry.MustRegister(starts)
+	legacyregistry.MustRegister(latencies)
+	legacyregistry.MustRegister(failures)
+	return &DialMetrics{starts: starts, latencies: latencies, failures: failures, clock: clock.RealClock{}}
+}
+
+// Clock returns the clock.
+func (m *DialMetrics) Clock() clock.Clock {
+	return m.clock
+}
+
+// SetClock sets the clock.
+func (m *DialMetrics) SetClock(c clock.Clock) {
+	m.clock = c
+}
+
+// Reset resets the metrics.
+func (m *DialMetrics) Reset() {
+	m.starts.Reset()
+	m.latencies.Reset()
+	m.failures.Reset()
+}
+
+// ObserveDialStart records the start of a dial attempt, labeled by protocol, transport.
+func (m *DialMetrics) ObserveDialStart(protocol, transport string) {
+	m.starts.WithLabelValues(protocol, transport).Inc()
+}
+
+// ObserveDialLatency records the latency of a dial, labeled by protocol, transport.
+func (m *DialMetrics) ObserveDialLatency(elapsed time.Duration, protocol, transport string) {
+	m.latencies.WithLabelValues(protocol, transport).Observe(elapsed.Seconds())
+}
+
+// ObserveDialFailure records a failed dial, labeled by protocol, transport, and the stage the dial failed at.
+func (m *DialMetrics) ObserveDialFailure(protocol, transport, stage string) {
+	m.failures.WithLabelValues(protocol, transport, stage).Inc()
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/OWNERS

@@ -0,0 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts
+  - dims

e2e/vendor/k8s.io/apiserver/pkg/server/filters/content_type.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import "net/http"
+
+// WithContentType sets both the Content-Type and the X-Content-Type-Options (nosniff) header
+func WithContentType(handler http.Handler, contentType string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", contentType)
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		handler.ServeHTTP(w, r)
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/cors.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"regexp"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+// TODO: use restful.CrossOriginResourceSharing
+// See github.com/emicklei/go-restful/blob/master/examples/cors/restful-CORS-filter.go, and
+// github.com/emicklei/go-restful/blob/master/examples/basicauth/restful-basic-authentication.go
+// Or, for a more detailed implementation use https://github.com/martini-contrib/cors
+// or implement CORS at your proxy layer.
+
+// WithCORS is a simple CORS implementation that wraps an http Handler.
+// Pass nil for allowedMethods and allowedHeaders to use the defaults. If allowedOriginPatterns
+// is empty or nil, no CORS support is installed.
+func WithCORS(handler http.Handler, allowedOriginPatterns []string, allowedMethods []string, allowedHeaders []string, exposedHeaders []string, allowCredentials string) http.Handler {
+	if len(allowedOriginPatterns) == 0 {
+		return handler
+	}
+	allowedOriginPatternsREs := allowedOriginRegexps(allowedOriginPatterns)
+
+	// Set defaults for methods and headers if nothing was passed
+	if allowedMethods == nil {
+		allowedMethods = []string{"POST", "GET", "OPTIONS", "PUT", "DELETE", "PATCH"}
+	}
+	allowMethodsResponseHeader := strings.Join(allowedMethods, ", ")
+
+	if allowedHeaders == nil {
+		allowedHeaders = []string{"Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization", "X-Requested-With", "If-Modified-Since"}
+	}
+	allowHeadersResponseHeader := strings.Join(allowedHeaders, ", ")
+
+	if exposedHeaders == nil {
+		exposedHeaders = []string{"Date"}
+	}
+	exposeHeadersResponseHeader := strings.Join(exposedHeaders, ", ")
+
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		origin := req.Header.Get("Origin")
+		if origin == "" {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		if !isOriginAllowed(origin, allowedOriginPatternsREs) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		w.Header().Set("Access-Control-Allow-Origin", origin)
+		w.Header().Set("Access-Control-Allow-Methods", allowMethodsResponseHeader)
+		w.Header().Set("Access-Control-Allow-Headers", allowHeadersResponseHeader)
+		w.Header().Set("Access-Control-Expose-Headers", exposeHeadersResponseHeader)
+		w.Header().Set("Access-Control-Allow-Credentials", allowCredentials)
+
+		// Stop here if its a preflight OPTIONS request
+		if req.Method == "OPTIONS" {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+
+		// Dispatch to the next handler
+		handler.ServeHTTP(w, req)
+	})
+}
+
+// isOriginAllowed returns true if the given origin header in the
+// request is allowed CORS.
+//
+// From https://www.rfc-editor.org/rfc/rfc6454#page-13
+//
+//	 a) The origin header can contain host and/or port
+//			serialized-origin   = scheme "://" host [ ":" port ]
+//
+//	 b) In some cases, a number of origins contribute to causing the user
+//	 agents to issue an HTTP request.  In those cases, the user agent MAY
+//	 list all the origins in the Origin header field. For example, if the
+//	 HTTP request was initially issued by one origin but then later
+//	 redirected by another origin, the user agent MAY inform the server
+//	 that two origins were involved in causing the user agent to issue the
+//	 request
+//			origin-list = serialized-origin *( SP serialized-origin )
+func isOriginAllowed(originHeader string, allowedOriginPatternsREs []*regexp.Regexp) bool {
+	for _, re := range allowedOriginPatternsREs {
+		if re.MatchString(originHeader) {
+			return true
+		}
+	}
+	return false
+}
+
+func allowedOriginRegexps(allowedOrigins []string) []*regexp.Regexp {
+	res, err := compileRegexps(allowedOrigins)
+	if err != nil {
+		klog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(allowedOrigins, ","), err)
+	}
+	return res
+}
+
+// Takes a list of strings and compiles them into a list of regular expressions
+func compileRegexps(regexpStrings []string) ([]*regexp.Regexp, error) {
+	regexps := []*regexp.Regexp{}
+	for _, regexpStr := range regexpStrings {
+		r, err := regexp.Compile(regexpStr)
+		if err != nil {
+			return []*regexp.Regexp{}, err
+		}
+		regexps = append(regexps, r)
+	}
+	return regexps, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package filters contains all the http handler chain filters which
+// are not api related.
+package filters // import "k8s.io/apiserver/pkg/server/filters"

e2e/vendor/k8s.io/apiserver/pkg/server/filters/goaway.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"math/rand"
+	"net/http"
+	"sync"
+)
+
+// GoawayDecider decides if server should send a GOAWAY
+type GoawayDecider interface {
+	Goaway(r *http.Request) bool
+}
+
+var (
+	// randPool used to get a rand.Rand and generate a random number thread-safely,
+	// which improve the performance of using rand.Rand with a locker
+	randPool = &sync.Pool{
+		New: func() interface{} {
+			return rand.New(rand.NewSource(rand.Int63()))
+		},
+	}
+)
+
+// WithProbabilisticGoaway returns an http.Handler that send GOAWAY probabilistically
+// according to the given chance for HTTP2 requests. After client receive GOAWAY,
+// the in-flight long-running requests will not be influenced, and the new requests
+// will use a new TCP connection to re-balancing to another server behind the load balance.
+func WithProbabilisticGoaway(inner http.Handler, chance float64) http.Handler {
+	return &goaway{
+		handler: inner,
+		decider: &probabilisticGoawayDecider{
+			chance: chance,
+			next: func() float64 {
+				rnd := randPool.Get().(*rand.Rand)
+				ret := rnd.Float64()
+				randPool.Put(rnd)
+				return ret
+			},
+		},
+	}
+}
+
+// goaway send a GOAWAY to client according to decider for HTTP2 requests
+type goaway struct {
+	handler http.Handler
+	decider GoawayDecider
+}
+
+// ServeHTTP implement HTTP handler
+func (h *goaway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Proto == "HTTP/2.0" && h.decider.Goaway(r) {
+		// Send a GOAWAY and tear down the TCP connection when idle.
+		w.Header().Set("Connection", "close")
+	}
+
+	h.handler.ServeHTTP(w, r)
+}
+
+// probabilisticGoawayDecider send GOAWAY probabilistically according to chance
+type probabilisticGoawayDecider struct {
+	chance float64
+	next   func() float64
+}
+
+// Goaway implement GoawayDecider
+func (p *probabilisticGoawayDecider) Goaway(r *http.Request) bool {
+	return p.next() < p.chance
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/hsts.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+)
+
+// WithHSTS is a simple HSTS implementation that wraps an http Handler.
+// If hstsDirectives is empty or nil, no HSTS support is installed.
+func WithHSTS(handler http.Handler, hstsDirectives []string) http.Handler {
+	if len(hstsDirectives) == 0 {
+		return handler
+	}
+	allDirectives := strings.Join(hstsDirectives, "; ")
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		// Chrome and Mozilla Firefox maintain an HSTS preload list
+		// issue : golang.org/issue/26162
+		// Set the Strict-Transport-Security header if it is not already set
+		if _, ok := w.Header()["Strict-Transport-Security"]; !ok {
+			w.Header().Set("Strict-Transport-Security", allDirectives)
+		}
+		handler.ServeHTTP(w, req)
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/longrunning.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// BasicLongRunningRequestCheck returns true if the given request has one of the specified verbs or one of the specified subresources, or is a profiler request.
+func BasicLongRunningRequestCheck(longRunningVerbs, longRunningSubresources sets.String) apirequest.LongRunningRequestCheck {
+	return func(r *http.Request, requestInfo *apirequest.RequestInfo) bool {
+		if longRunningVerbs.Has(requestInfo.Verb) {
+			return true
+		}
+		if requestInfo.IsResourceRequest && longRunningSubresources.Has(requestInfo.Subresource) {
+			return true
+		}
+		if !requestInfo.IsResourceRequest && strings.HasPrefix(requestInfo.Path, "/debug/pprof/") {
+			return true
+		}
+		return false
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -0,0 +1,222 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+
+	"k8s.io/klog/v2"
+)
+
+const (
+	// Constant for the retry-after interval on rate limiting.
+	retryAfter = "1"
+
+	// How often inflight usage metric should be updated. Because
+	// the metrics tracks maximal value over period making this
+	// longer will increase the metric value.
+	inflightUsageMetricUpdatePeriod = time.Second
+)
+
+var (
+	nonMutatingRequestVerbs = sets.NewString("get", "list", "watch")
+	watchVerbs              = sets.NewString("watch")
+)
+
+func handleError(w http.ResponseWriter, r *http.Request, err error) {
+	errorMsg := fmt.Sprintf("Internal Server Error: %#v", r.RequestURI)
+	http.Error(w, errorMsg, http.StatusInternalServerError)
+	klog.Error(err.Error())
+}
+
+// requestWatermark is used to track maximal numbers of requests in a particular phase of handling
+type requestWatermark struct {
+	phase                                string
+	readOnlyObserver, mutatingObserver   fcmetrics.RatioedGauge
+	lock                                 sync.Mutex
+	readOnlyWatermark, mutatingWatermark int
+}
+
+func (w *requestWatermark) recordMutating(mutatingVal int) {
+	w.mutatingObserver.Set(float64(mutatingVal))
+
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.mutatingWatermark < mutatingVal {
+		w.mutatingWatermark = mutatingVal
+	}
+}
+
+func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
+	w.readOnlyObserver.Set(float64(readOnlyVal))
+
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.readOnlyWatermark < readOnlyVal {
+		w.readOnlyWatermark = readOnlyVal
+	}
+}
+
+// watermark tracks requests being executed (not waiting in a queue)
+var watermark = &requestWatermark{
+	phase: metrics.ExecutingPhase,
+}
+
+// startWatermarkMaintenance starts the goroutines to observe and maintain the specified watermark.
+func startWatermarkMaintenance(watermark *requestWatermark, stopCh <-chan struct{}) {
+	// Periodically update the inflight usage metric.
+	go wait.Until(func() {
+		watermark.lock.Lock()
+		readOnlyWatermark := watermark.readOnlyWatermark
+		mutatingWatermark := watermark.mutatingWatermark
+		watermark.readOnlyWatermark = 0
+		watermark.mutatingWatermark = 0
+		watermark.lock.Unlock()
+
+		metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
+	}, inflightUsageMetricUpdatePeriod, stopCh)
+}
+
+var initMaxInFlightOnce sync.Once
+
+func initMaxInFlight(nonMutatingLimit, mutatingLimit int) {
+	initMaxInFlightOnce.Do(func() {
+		// Fetching these gauges is delayed until after their underlying metric has been registered
+		// so that this latches onto the efficient implementation.
+		watermark.readOnlyObserver = fcmetrics.GetExecutingReadonlyConcurrency()
+		watermark.mutatingObserver = fcmetrics.GetExecutingMutatingConcurrency()
+		if nonMutatingLimit != 0 {
+			watermark.readOnlyObserver.SetDenominator(float64(nonMutatingLimit))
+			klog.V(2).InfoS("Set denominator for readonly requests", "limit", nonMutatingLimit)
+		}
+		if mutatingLimit != 0 {
+			watermark.mutatingObserver.SetDenominator(float64(mutatingLimit))
+			klog.V(2).InfoS("Set denominator for mutating requests", "limit", mutatingLimit)
+		}
+	})
+}
+
+// WithMaxInFlightLimit limits the number of in-flight requests to buffer size of the passed in channel.
+func WithMaxInFlightLimit(
+	handler http.Handler,
+	nonMutatingLimit int,
+	mutatingLimit int,
+	longRunningRequestCheck apirequest.LongRunningRequestCheck,
+) http.Handler {
+	if nonMutatingLimit == 0 && mutatingLimit == 0 {
+		return handler
+	}
+	var nonMutatingChan chan bool
+	var mutatingChan chan bool
+	if nonMutatingLimit != 0 {
+		nonMutatingChan = make(chan bool, nonMutatingLimit)
+		klog.V(2).InfoS("Initialized nonMutatingChan", "len", nonMutatingLimit)
+	} else {
+		klog.V(2).InfoS("Running with nil nonMutatingChan")
+	}
+	if mutatingLimit != 0 {
+		mutatingChan = make(chan bool, mutatingLimit)
+		klog.V(2).InfoS("Initialized mutatingChan", "len", mutatingLimit)
+	} else {
+		klog.V(2).InfoS("Running with nil mutatingChan")
+	}
+	initMaxInFlight(nonMutatingLimit, mutatingLimit)
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			handleError(w, r, fmt.Errorf("no RequestInfo found in context, handler chain must be wrong"))
+			return
+		}
+
+		// Skip tracking long running events.
+		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) {
+			handler.ServeHTTP(w, r)
+			return
+		}
+
+		var c chan bool
+		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
+		if isMutatingRequest {
+			c = mutatingChan
+		} else {
+			c = nonMutatingChan
+		}
+
+		if c == nil {
+			handler.ServeHTTP(w, r)
+		} else {
+
+			select {
+			case c <- true:
+				// We note the concurrency level both while the
+				// request is being served and after it is done being
+				// served, because both states contribute to the
+				// sampled stats on concurrency.
+				if isMutatingRequest {
+					watermark.recordMutating(len(c))
+				} else {
+					watermark.recordReadOnly(len(c))
+				}
+				defer func() {
+					<-c
+					if isMutatingRequest {
+						watermark.recordMutating(len(c))
+					} else {
+						watermark.recordReadOnly(len(c))
+					}
+				}()
+				handler.ServeHTTP(w, r)
+
+			default:
+				// at this point we're about to return a 429, BUT not all actors should be rate limited.  A system:master is so powerful
+				// that they should always get an answer.  It's a super-admin or a loopback connection.
+				if currUser, ok := apirequest.UserFrom(ctx); ok {
+					for _, group := range currUser.GetGroups() {
+						if group == user.SystemPrivilegedGroup {
+							handler.ServeHTTP(w, r)
+							return
+						}
+					}
+				}
+				// We need to split this data between buckets used for throttling.
+				metrics.RecordDroppedRequest(r, requestInfo, metrics.APIServerComponent, isMutatingRequest)
+				metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests)
+				tooManyRequests(r, w, retryAfter)
+			}
+		}
+	})
+}
+
+// StartMaxInFlightWatermarkMaintenance starts the goroutines to observe and maintain watermarks for max-in-flight
+// requests.
+func StartMaxInFlightWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -0,0 +1,426 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"runtime"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	flowcontrol "k8s.io/api/flowcontrol/v1"
+	apitypes "k8s.io/apimachinery/pkg/types"
+	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server/httplog"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	"k8s.io/klog/v2"
+	utilsclock "k8s.io/utils/clock"
+)
+
+// PriorityAndFairnessClassification identifies the results of
+// classification for API Priority and Fairness
+type PriorityAndFairnessClassification struct {
+	FlowSchemaName    string
+	FlowSchemaUID     apitypes.UID
+	PriorityLevelName string
+	PriorityLevelUID  apitypes.UID
+}
+
+// waitingMark tracks requests waiting rather than being executed
+var waitingMark = &requestWatermark{
+	phase: epmetrics.WaitingPhase,
+}
+
+var atomicMutatingExecuting, atomicReadOnlyExecuting int32
+var atomicMutatingWaiting, atomicReadOnlyWaiting int32
+
+// newInitializationSignal is defined for testing purposes.
+var newInitializationSignal = utilflowcontrol.NewInitializationSignal
+
+func truncateLogField(s string) string {
+	const maxFieldLogLength = 64
+
+	if len(s) > maxFieldLogLength {
+		s = s[0:maxFieldLogLength]
+	}
+	return s
+}
+
+var initAPFOnce sync.Once
+
+type priorityAndFairnessHandler struct {
+	handler                 http.Handler
+	longRunningRequestCheck apirequest.LongRunningRequestCheck
+	fcIfc                   utilflowcontrol.Interface
+	workEstimator           flowcontrolrequest.WorkEstimatorFunc
+
+	// droppedRequests tracks the history of dropped requests for
+	// the purpose of computing RetryAfter header to avoid system
+	// overload.
+	droppedRequests utilflowcontrol.DroppedRequestsTracker
+
+	// newReqWaitCtxFn creates a derived context with a deadline
+	// of how long a given request can wait in its queue.
+	newReqWaitCtxFn func(context.Context) (context.Context, context.CancelFunc)
+}
+
+func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+	if !ok {
+		handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
+		return
+	}
+	user, ok := apirequest.UserFrom(ctx)
+	if !ok {
+		handleError(w, r, fmt.Errorf("no User found in context"))
+		return
+	}
+
+	isWatchRequest := watchVerbs.Has(requestInfo.Verb)
+
+	// Skip tracking long running non-watch requests.
+	if h.longRunningRequestCheck != nil && h.longRunningRequestCheck(r, requestInfo) && !isWatchRequest {
+		klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+
+	var classification *PriorityAndFairnessClassification
+	noteFn := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string) {
+		classification = &PriorityAndFairnessClassification{
+			FlowSchemaName:    fs.Name,
+			FlowSchemaUID:     fs.UID,
+			PriorityLevelName: pl.Name,
+			PriorityLevelUID:  pl.UID,
+		}
+
+		httplog.AddKeyValue(ctx, "apf_pl", truncateLogField(pl.Name))
+		httplog.AddKeyValue(ctx, "apf_fs", truncateLogField(fs.Name))
+	}
+	// estimateWork is called, if at all, after noteFn
+	estimateWork := func() flowcontrolrequest.WorkEstimate {
+		if classification == nil {
+			// workEstimator is being invoked before classification of
+			// the request has completed, we should never be here though.
+			klog.ErrorS(fmt.Errorf("workEstimator is being invoked before classification of the request has completed"),
+				"Using empty FlowSchema and PriorityLevelConfiguration name", "verb", r.Method, "URI", r.RequestURI)
+			return h.workEstimator(r, "", "")
+		}
+
+		workEstimate := h.workEstimator(r, classification.FlowSchemaName, classification.PriorityLevelName)
+
+		fcmetrics.ObserveWorkEstimatedSeats(classification.PriorityLevelName, classification.FlowSchemaName, workEstimate.MaxSeats())
+		httplog.AddKeyValue(ctx, "apf_iseats", workEstimate.InitialSeats)
+		httplog.AddKeyValue(ctx, "apf_fseats", workEstimate.FinalSeats)
+		httplog.AddKeyValue(ctx, "apf_additionalLatency", workEstimate.AdditionalLatency)
+
+		return workEstimate
+	}
+
+	var served bool
+	isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
+	noteExecutingDelta := func(delta int32) {
+		if isMutatingRequest {
+			watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
+		} else {
+			watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
+		}
+	}
+	noteWaitingDelta := func(delta int32) {
+		if isMutatingRequest {
+			waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
+		} else {
+			waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
+		}
+	}
+	queueNote := func(inQueue bool) {
+		if inQueue {
+			noteWaitingDelta(1)
+		} else {
+			noteWaitingDelta(-1)
+		}
+	}
+
+	digest := utilflowcontrol.RequestDigest{
+		RequestInfo: requestInfo,
+		User:        user,
+	}
+
+	if isWatchRequest {
+		// This channel blocks calling handler.ServeHTTP() until closed, and is closed inside execute().
+		// If APF rejects the request, it is never closed.
+		shouldStartWatchCh := make(chan struct{})
+
+		watchInitializationSignal := newInitializationSignal()
+		// This wraps the request passed to handler.ServeHTTP(),
+		// setting a context that plumbs watchInitializationSignal to storage
+		var watchReq *http.Request
+		// This is set inside execute(), prior to closing shouldStartWatchCh.
+		// If the request is rejected by APF it is left nil.
+		var forgetWatch utilflowcontrol.ForgetWatchFunc
+
+		defer func() {
+			// Protect from the situation when request will not reach storage layer
+			// and the initialization signal will not be send.
+			if watchInitializationSignal != nil {
+				watchInitializationSignal.Signal()
+			}
+			// Forget the watcher if it was registered.
+			//
+			// This is race-free because by this point, one of the following occurred:
+			// case <-shouldStartWatchCh: execute() completed the assignment to forgetWatch
+			// case <-resultCh: Handle() completed, and Handle() does not return
+			//   while execute() is running
+			if forgetWatch != nil {
+				forgetWatch()
+			}
+		}()
+
+		execute := func() {
+			startedAt := time.Now()
+			defer func() {
+				httplog.AddKeyValue(ctx, "apf_init_latency", time.Since(startedAt))
+			}()
+			noteExecutingDelta(1)
+			defer noteExecutingDelta(-1)
+			served = true
+			setResponseHeaders(classification, w)
+
+			forgetWatch = h.fcIfc.RegisterWatch(r)
+
+			// Notify the main thread that we're ready to start the watch.
+			close(shouldStartWatchCh)
+
+			// Wait until the request is finished from the APF point of view
+			// (which is when its initialization is done).
+			watchInitializationSignal.Wait()
+		}
+
+		// Ensure that an item can be put to resultCh asynchronously.
+		resultCh := make(chan interface{}, 1)
+
+		// Call Handle in a separate goroutine.
+		// The reason for it is that from APF point of view, the request processing
+		// finishes as soon as watch is initialized (which is generally orders of
+		// magnitude faster then the watch request itself). This means that Handle()
+		// call finishes much faster and for performance reasons we want to reduce
+		// the number of running goroutines - so we run the shorter thing in a
+		// dedicated goroutine and the actual watch handler in the main one.
+		go func() {
+			defer func() {
+				err := recover()
+				// do not wrap the sentinel ErrAbortHandler panic value
+				if err != nil && err != http.ErrAbortHandler {
+					// Same as stdlib http server code. Manually allocate stack
+					// trace buffer size to prevent excessively large logs
+					const size = 64 << 10
+					buf := make([]byte, size)
+					buf = buf[:runtime.Stack(buf, false)]
+					err = fmt.Sprintf("%v\n%s", err, buf)
+				}
+
+				// Ensure that the result is put into resultCh independently of the panic.
+				resultCh <- err
+			}()
+
+			// We create handleCtx with an adjusted deadline, for two reasons.
+			// One is to limit the time the request waits before its execution starts.
+			// The other reason for it is that Handle() underneath may start additional goroutine
+			// that is blocked on context cancellation. However, from APF point of view,
+			// we don't want to wait until the whole watch request is processed (which is
+			// when it context is actually cancelled) - we want to unblock the goroutine as
+			// soon as the request is processed from the APF point of view.
+			//
+			// Note that we explicitly do NOT call the actuall handler using that context
+			// to avoid cancelling request too early.
+			handleCtx, handleCtxCancel := h.newReqWaitCtxFn(ctx)
+			defer handleCtxCancel()
+
+			// Note that Handle will return irrespective of whether the request
+			// executes or is rejected. In the latter case, the function will return
+			// without calling the passed `execute` function.
+			h.fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
+		}()
+
+		select {
+		case <-shouldStartWatchCh:
+			func() {
+				// TODO: if both goroutines panic, propagate the stack traces from both
+				// goroutines so they are logged properly:
+				defer func() {
+					// Protect from the situation when request will not reach storage layer
+					// and the initialization signal will not be send.
+					// It has to happen before waiting on the resultCh below.
+					watchInitializationSignal.Signal()
+					// TODO: Consider finishing the request as soon as Handle call panics.
+					if err := <-resultCh; err != nil {
+						panic(err)
+					}
+				}()
+				watchCtx := utilflowcontrol.WithInitializationSignal(ctx, watchInitializationSignal)
+				watchReq = r.WithContext(watchCtx)
+				h.handler.ServeHTTP(w, watchReq)
+			}()
+		case err := <-resultCh:
+			if err != nil {
+				panic(err)
+			}
+		}
+	} else {
+		execute := func() {
+			noteExecutingDelta(1)
+			defer noteExecutingDelta(-1)
+			served = true
+			setResponseHeaders(classification, w)
+
+			h.handler.ServeHTTP(w, r)
+		}
+
+		func() {
+			handleCtx, cancelFn := h.newReqWaitCtxFn(ctx)
+			defer cancelFn()
+			h.fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
+		}()
+	}
+
+	if !served {
+		setResponseHeaders(classification, w)
+
+		epmetrics.RecordDroppedRequest(r, requestInfo, epmetrics.APIServerComponent, isMutatingRequest)
+		epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
+		h.droppedRequests.RecordDroppedRequest(classification.PriorityLevelName)
+
+		// TODO(wojtek-t): Idea from deads2k: we can consider some jittering and in case of non-int
+		//  number, just return the truncated result and sleep the remainder server-side.
+		tooManyRequests(r, w, strconv.Itoa(int(h.droppedRequests.GetRetryAfter(classification.PriorityLevelName))))
+	}
+}
+
+// WithPriorityAndFairness limits the number of in-flight
+// requests in a fine-grained way.
+func WithPriorityAndFairness(
+	handler http.Handler,
+	longRunningRequestCheck apirequest.LongRunningRequestCheck,
+	fcIfc utilflowcontrol.Interface,
+	workEstimator flowcontrolrequest.WorkEstimatorFunc,
+	defaultRequestWaitLimit time.Duration,
+) http.Handler {
+	if fcIfc == nil {
+		klog.Warningf("priority and fairness support not found, skipping")
+		return handler
+	}
+	initAPFOnce.Do(func() {
+		initMaxInFlight(0, 0)
+		// Fetching these gauges is delayed until after their underlying metric has been registered
+		// so that this latches onto the efficient implementation.
+		waitingMark.readOnlyObserver = fcmetrics.GetWaitingReadonlyConcurrency()
+		waitingMark.mutatingObserver = fcmetrics.GetWaitingMutatingConcurrency()
+	})
+
+	clock := &utilsclock.RealClock{}
+	newReqWaitCtxFn := func(ctx context.Context) (context.Context, context.CancelFunc) {
+		return getRequestWaitContext(ctx, defaultRequestWaitLimit, clock)
+	}
+
+	priorityAndFairnessHandler := &priorityAndFairnessHandler{
+		handler:                 handler,
+		longRunningRequestCheck: longRunningRequestCheck,
+		fcIfc:                   fcIfc,
+		workEstimator:           workEstimator,
+		droppedRequests:         utilflowcontrol.NewDroppedRequestsTracker(),
+		newReqWaitCtxFn:         newReqWaitCtxFn,
+	}
+	return http.HandlerFunc(priorityAndFairnessHandler.Handle)
+}
+
+// StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
+// priority-and-fairness requests.
+func StartPriorityAndFairnessWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+	startWatermarkMaintenance(waitingMark, stopCh)
+}
+
+func setResponseHeaders(classification *PriorityAndFairnessClassification, w http.ResponseWriter) {
+	if classification == nil {
+		return
+	}
+
+	// We intentionally set the UID of the flow-schema and priority-level instead of name. This is so that
+	// the names that cluster-admins choose for categorization and priority levels are not exposed, also
+	// the names might make it obvious to the users that they are rejected due to classification with low priority.
+	w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
+	w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
+}
+
+func tooManyRequests(req *http.Request, w http.ResponseWriter, retryAfter string) {
+	// Return a 429 status indicating "Too Many Requests"
+	w.Header().Set("Retry-After", retryAfter)
+	http.Error(w, "Too many requests, please try again later.", http.StatusTooManyRequests)
+}
+
+// getRequestWaitContext returns a new context with a deadline of how
+// long the request is allowed to wait before it is removed from its
+// queue and rejected.
+// The context.CancelFunc returned must never be nil and the caller is
+// responsible for calling the CancelFunc function for cleanup.
+//   - ctx: the context associated with the request (it may or may
+//     not have a deadline).
+//   - defaultRequestWaitLimit: the default wait duration that is used
+//     if the request context does not have any deadline.
+//     (a) initialization of a watch or
+//     (b) a request whose context has no deadline
+//
+// clock comes in handy for testing the function
+func getRequestWaitContext(ctx context.Context, defaultRequestWaitLimit time.Duration, clock utilsclock.PassiveClock) (context.Context, context.CancelFunc) {
+	if ctx.Err() != nil {
+		return ctx, func() {}
+	}
+
+	reqArrivedAt := clock.Now()
+	if reqReceivedTimestamp, ok := apirequest.ReceivedTimestampFrom(ctx); ok {
+		reqArrivedAt = reqReceivedTimestamp
+	}
+
+	// a) we will allow the request to wait in the queue for one
+	// fourth of the time of its allotted deadline.
+	// b) if the request context does not have any deadline
+	// then we default to 'defaultRequestWaitLimit'
+	// in any case, the wait limit for any request must not
+	// exceed the hard limit of 1m
+	//
+	// request has deadline:
+	//   wait-limit = min(remaining deadline / 4, 1m)
+	// request has no deadline:
+	//   wait-limit = min(defaultRequestWaitLimit, 1m)
+	thisReqWaitLimit := defaultRequestWaitLimit
+	if deadline, ok := ctx.Deadline(); ok {
+		thisReqWaitLimit = deadline.Sub(reqArrivedAt) / 4
+	}
+	if thisReqWaitLimit > time.Minute {
+		thisReqWaitLimit = time.Minute
+	}
+
+	return context.WithDeadline(ctx, reqArrivedAt.Add(thisReqWaitLimit))
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -0,0 +1,308 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/server/httplog"
+)
+
+// WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
+func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck) http.Handler {
+	if longRunning == nil {
+		return handler
+	}
+	timeoutFunc := func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
+		// TODO unify this with apiserver.MaxInFlightLimit
+		ctx := req.Context()
+
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			// if this happens, the handler chain isn't setup correctly because there is no request info
+			return req, false, func() {}, apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout"))
+		}
+
+		if longRunning(req, requestInfo) {
+			return req, true, nil, nil
+		}
+
+		postTimeoutFn := func() {
+			metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
+		}
+		return req, false, postTimeoutFn, apierrors.NewTimeoutError("request did not complete within the allotted timeout", 0)
+	}
+	return WithTimeout(handler, timeoutFunc)
+}
+
+type timeoutFunc = func(*http.Request) (req *http.Request, longRunning bool, postTimeoutFunc func(), err *apierrors.StatusError)
+
+// WithTimeout returns an http.Handler that runs h with a timeout
+// determined by timeoutFunc. The new http.Handler calls h.ServeHTTP to handle
+// each request, but if a call runs for longer than its time limit, the
+// handler responds with a 504 Gateway Timeout error and the message
+// provided. (If msg is empty, a suitable default message will be sent.) After
+// the handler times out, writes by h to its http.ResponseWriter will return
+// http.ErrHandlerTimeout. If timeoutFunc returns a nil timeout channel, no
+// timeout will be enforced. recordFn is a function that will be invoked whenever
+// a timeout happens.
+func WithTimeout(h http.Handler, timeoutFunc timeoutFunc) http.Handler {
+	return &timeoutHandler{h, timeoutFunc}
+}
+
+type timeoutHandler struct {
+	handler http.Handler
+	timeout timeoutFunc
+}
+
+func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	r, longRunning, postTimeoutFn, err := t.timeout(r)
+	if longRunning {
+		t.handler.ServeHTTP(w, r)
+		return
+	}
+
+	timeoutCh := r.Context().Done()
+
+	// resultCh is used as both errCh and stopCh
+	resultCh := make(chan interface{})
+	var tw timeoutWriter
+	tw, w = newTimeoutWriter(w)
+
+	// Make a copy of request and work on it in new goroutine
+	// to avoid race condition when accessing/modifying request (e.g. headers)
+	rCopy := r.Clone(r.Context())
+	go func() {
+		defer func() {
+			err := recover()
+			// do not wrap the sentinel ErrAbortHandler panic value
+			if err != nil && err != http.ErrAbortHandler {
+				// Same as stdlib http server code. Manually allocate stack
+				// trace buffer size to prevent excessively large logs
+				const size = 64 << 10
+				buf := make([]byte, size)
+				buf = buf[:runtime.Stack(buf, false)]
+				err = fmt.Sprintf("%v\n%s", err, buf)
+			}
+			resultCh <- err
+		}()
+		t.handler.ServeHTTP(w, rCopy)
+	}()
+	select {
+	case err := <-resultCh:
+		// panic if error occurs; stop otherwise
+		if err != nil {
+			panic(err)
+		}
+		return
+	case <-timeoutCh:
+		defer func() {
+			// resultCh needs to have a reader, since the function doing
+			// the work needs to send to it. This is defer'd to ensure it runs
+			// ever if the post timeout work itself panics.
+			go func() {
+				timedOutAt := time.Now()
+				res := <-resultCh
+
+				status := metrics.PostTimeoutHandlerOK
+				if res != nil {
+					// a non nil res indicates that there was a panic.
+					status = metrics.PostTimeoutHandlerPanic
+				}
+
+				metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status)
+				utilruntime.HandleErrorWithContext(r.Context(), nil, "Post-timeout activity", "timeElapsed", time.Since(timedOutAt), "method", r.Method, "path", r.URL.Path, "result", res)
+			}()
+		}()
+		httplog.SetStacktracePredicate(r.Context(), func(status int) bool {
+			return false
+		})
+		defer postTimeoutFn()
+		tw.timeout(err)
+	}
+}
+
+type timeoutWriter interface {
+	http.ResponseWriter
+	timeout(*apierrors.StatusError)
+}
+
+func newTimeoutWriter(w http.ResponseWriter) (timeoutWriter, http.ResponseWriter) {
+	base := &baseTimeoutWriter{w: w, handlerHeaders: w.Header().Clone()}
+	wrapped := responsewriter.WrapForHTTP1Or2(base)
+
+	return base, wrapped
+}
+
+var _ http.ResponseWriter = &baseTimeoutWriter{}
+var _ responsewriter.UserProvidedDecorator = &baseTimeoutWriter{}
+
+type baseTimeoutWriter struct {
+	w http.ResponseWriter
+
+	// headers written by the normal handler
+	handlerHeaders http.Header
+
+	mu sync.Mutex
+	// if the timeout handler has timeout
+	timedOut bool
+	// if this timeout writer has wrote header
+	wroteHeader bool
+	// if this timeout writer has been hijacked
+	hijacked bool
+}
+
+func (tw *baseTimeoutWriter) Unwrap() http.ResponseWriter {
+	return tw.w
+}
+
+func (tw *baseTimeoutWriter) Header() http.Header {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return http.Header{}
+	}
+
+	return tw.handlerHeaders
+}
+
+func (tw *baseTimeoutWriter) Write(p []byte) (int, error) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return 0, http.ErrHandlerTimeout
+	}
+	if tw.hijacked {
+		return 0, http.ErrHijacked
+	}
+
+	if !tw.wroteHeader {
+		copyHeaders(tw.w.Header(), tw.handlerHeaders)
+		tw.wroteHeader = true
+	}
+	return tw.w.Write(p)
+}
+
+func (tw *baseTimeoutWriter) Flush() {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Flusher if the inner object (tw.w) implements http.Flusher.
+	tw.w.(http.Flusher).Flush()
+}
+
+func (tw *baseTimeoutWriter) WriteHeader(code int) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut || tw.wroteHeader || tw.hijacked {
+		return
+	}
+
+	copyHeaders(tw.w.Header(), tw.handlerHeaders)
+	tw.wroteHeader = true
+	tw.w.WriteHeader(code)
+}
+
+func copyHeaders(dst, src http.Header) {
+	for k, v := range src {
+		dst[k] = v
+	}
+}
+
+func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	tw.timedOut = true
+
+	// The timeout writer has not been used by the inner handler.
+	// We can safely timeout the HTTP request by sending by a timeout
+	// handler
+	if !tw.wroteHeader && !tw.hijacked {
+		tw.w.WriteHeader(http.StatusGatewayTimeout)
+		enc := json.NewEncoder(tw.w)
+		enc.Encode(&err.ErrStatus)
+	} else {
+		// The timeout writer has been used by the inner handler. There is
+		// no way to timeout the HTTP request at the point. We have to shutdown
+		// the connection for HTTP1 or reset stream for HTTP2.
+		//
+		// Note from the golang's docs:
+		// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+		// that the effect of the panic was isolated to the active request.
+		// It recovers the panic, logs a stack trace to the server error log,
+		// and either closes the network connection or sends an HTTP/2
+		// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+		// the client sees an interrupted response but the server doesn't log
+		// an error, panic with the value ErrAbortHandler.
+		//
+		// We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs
+		panic(http.ErrAbortHandler)
+	}
+}
+
+func (tw *baseTimeoutWriter) CloseNotify() <-chan bool {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		done := make(chan bool)
+		close(done)
+		return done
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.CloseNotifier if the inner object (tw.w) implements http.CloseNotifier.
+	return tw.w.(http.CloseNotifier).CloseNotify()
+}
+
+func (tw *baseTimeoutWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return nil, nil, http.ErrHandlerTimeout
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Hijacker if the inner object (tw.w) implements http.Hijacker.
+	conn, rw, err := tw.w.(http.Hijacker).Hijack()
+	if err == nil {
+		tw.hijacked = true
+	}
+	return conn, rw, err
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/waitgroup.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// RequestWaitGroup helps with the accounting of request(s) that are in
+// flight: the caller is expected to invoke Add(1) before executing the
+// request handler and then invoke Done() when the handler finishes.
+// NOTE: implementations must ensure that it is thread-safe
+// when invoked from multiple goroutines.
+type RequestWaitGroup interface {
+	// Add adds delta, which may be negative, similar to sync.WaitGroup.
+	// If Add with a positive delta happens after Wait, it will return error,
+	// which prevent unsafe Add.
+	Add(delta int) error
+
+	// Done decrements the WaitGroup counter.
+	Done()
+}
+
+// WithWaitGroup adds all non long-running requests to wait group, which is used for graceful shutdown.
+func WithWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg RequestWaitGroup) http.Handler {
+	// NOTE: both WithWaitGroup and WithRetryAfter must use the same exact isRequestExemptFunc 'isRequestExemptFromRetryAfter,
+	// otherwise SafeWaitGroup might wait indefinitely and will prevent the server from shutting down gracefully.
+	return withWaitGroup(handler, longRunning, wg, isRequestExemptFromRetryAfter)
+}
+
+func withWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg RequestWaitGroup, isRequestExemptFn isRequestExemptFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			// if this happens, the handler chain isn't setup correctly because there is no request info
+			responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
+			return
+		}
+
+		if longRunning(req, requestInfo) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		if err := wg.Add(1); err != nil {
+			// shutdown delay duration has elapsed and SafeWaitGroup.Wait has been invoked,
+			// this means 'WithRetryAfter' has started sending Retry-After response.
+			// we are going to exempt the same set of requests that WithRetryAfter are
+			// exempting from being rejected with a Retry-After response.
+			if isRequestExemptFn(req) {
+				handler.ServeHTTP(w, req)
+				return
+			}
+
+			// When apiserver is shutting down, signal clients to retry
+			// There is a good chance the client hit a different server, so a tight retry is good for client responsiveness.
+			waitGroupWriteRetryAfterToResponse(w)
+			return
+		}
+
+		defer wg.Done()
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func waitGroupWriteRetryAfterToResponse(w http.ResponseWriter) {
+	w.Header().Add("Retry-After", "1")
+	w.Header().Set("Content-Type", runtime.ContentTypeJSON)
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	statusErr := apierrors.NewServiceUnavailable("apiserver is shutting down").Status()
+	w.WriteHeader(int(statusErr.Code))
+	fmt.Fprintln(w, runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), &statusErr))
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/watch_termination.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"errors"
+	"net/http"
+
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog/v2"
+)
+
+func WithWatchTerminationDuringShutdown(handler http.Handler, termination apirequest.ServerShutdownSignal, wg RequestWaitGroup) http.Handler {
+	if termination == nil || wg == nil {
+		klog.Warningf("watch termination during shutdown not attached to the handler chain")
+		return handler
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			// if this happens, the handler chain isn't setup correctly because there is no request info
+			responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
+			return
+		}
+		if !watchVerbs.Has(requestInfo.Verb) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		if err := wg.Add(1); err != nil {
+			// When apiserver is shutting down, signal clients to retry
+			// There is a good chance the client hit a different server, so a tight retry is good for client responsiveness.
+			waitGroupWriteRetryAfterToResponse(w)
+			return
+		}
+
+		// attach ServerShutdownSignal to the watch request so that the
+		// watch handler loop can return as soon as the server signals
+		// that it is shutting down.
+		ctx = apirequest.WithServerShutdownSignal(req.Context(), termination)
+		req = req.WithContext(ctx)
+
+		defer wg.Done()
+		handler.ServeHTTP(w, req)
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/with_retry_after.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+)
+
+var (
+	// health probes and metrics scraping are never rejected, we will continue
+	// serving these requests after shutdown delay duration elapses.
+	pathPrefixesExemptFromRetryAfter = []string{
+		"/readyz",
+		"/livez",
+		"/healthz",
+		"/metrics",
+	}
+)
+
+// isRequestExemptFunc returns true if the request should not be rejected,
+// with a Retry-After response, otherwise it returns false.
+type isRequestExemptFunc func(*http.Request) bool
+
+// retryAfterParams dictates how the Retry-After response is constructed
+type retryAfterParams struct {
+	// TearDownConnection is true when we should send a 'Connection: close'
+	// header in the response so net/http can tear down the TCP connection.
+	TearDownConnection bool
+
+	// Message describes why Retry-After response has been sent by the server
+	Message string
+}
+
+// shouldRespondWithRetryAfterFunc returns true if the requests should
+// be rejected with a Retry-After response once certain conditions are met.
+// The retryAfterParams returned contains instructions on how to
+// construct the Retry-After response.
+type shouldRespondWithRetryAfterFunc func() (*retryAfterParams, bool)
+
+// WithRetryAfter rejects any incoming new request(s) with a 429
+// if the specified shutdownDelayDurationElapsedFn channel is closed
+//
+// It includes new request(s) on a new or an existing TCP connection
+// Any new request(s) arriving after shutdownDelayDurationElapsedFn is closed
+// are replied with a 429 and the following response headers:
+//   - 'Retry-After: N` (so client can retry after N seconds, hopefully on a new apiserver instance)
+//   - 'Connection: close': tear down the TCP connection
+//
+// TODO: is there a way to merge WithWaitGroup and this filter?
+func WithRetryAfter(handler http.Handler, shutdownDelayDurationElapsedCh <-chan struct{}) http.Handler {
+	shutdownRetryAfterParams := &retryAfterParams{
+		TearDownConnection: true,
+		Message:            "The apiserver is shutting down, please try again later.",
+	}
+
+	// NOTE: both WithRetryAfter and WithWaitGroup must use the same exact isRequestExemptFunc 'isRequestExemptFromRetryAfter,
+	// otherwise SafeWaitGroup might wait indefinitely and will prevent the server from shutting down gracefully.
+	return withRetryAfter(handler, isRequestExemptFromRetryAfter, func() (*retryAfterParams, bool) {
+		select {
+		case <-shutdownDelayDurationElapsedCh:
+			return shutdownRetryAfterParams, true
+		default:
+			return nil, false
+		}
+	})
+}
+
+func withRetryAfter(handler http.Handler, isRequestExemptFn isRequestExemptFunc, shouldRespondWithRetryAfterFn shouldRespondWithRetryAfterFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		params, send := shouldRespondWithRetryAfterFn()
+		if !send || isRequestExemptFn(req) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		// If we are here this means it's time to send Retry-After response
+		//
+		// Copied from net/http2 library
+		// "Connection" headers aren't allowed in HTTP/2 (RFC 7540, 8.1.2.2),
+		// but respect "Connection" == "close" to mean sending a GOAWAY and tearing
+		// down the TCP connection when idle, like we do for HTTP/1.
+		if params.TearDownConnection {
+			w.Header().Set("Connection", "close")
+		}
+
+		// Return a 429 status asking the client to try again after 5 seconds
+		w.Header().Set("Retry-After", "5")
+		http.Error(w, params.Message, http.StatusTooManyRequests)
+	})
+}
+
+// isRequestExemptFromRetryAfter returns true if the given request should be exempt
+// from being rejected with a 'Retry-After' response.
+// NOTE: both 'WithRetryAfter' and 'WithWaitGroup' filters should use this function
+// to exempt the set of requests from being rejected or tracked.
+func isRequestExemptFromRetryAfter(r *http.Request) bool {
+	return isKubeApiserverUserAgent(r) || hasExemptPathPrefix(r)
+}
+
+// isKubeApiserverUserAgent returns true if the user-agent matches
+// the one set by the local loopback.
+// NOTE: we can't look up the authenticated user informaion from the
+// request context since the authentication filter has not executed yet.
+func isKubeApiserverUserAgent(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), "kube-apiserver/")
+}
+
+func hasExemptPathPrefix(r *http.Request) bool {
+	for _, whiteListedPrefix := range pathPrefixesExemptFromRetryAfter {
+		if strings.HasPrefix(r.URL.Path, whiteListedPrefix) {
+			return true
+		}
+	}
+	return false
+}

e2e/vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/klog/v2"
+)
+
+// WithPanicRecovery wraps an http Handler to recover and log panics (except in the special case of http.ErrAbortHandler panics, which suppress logging).
+func WithPanicRecovery(handler http.Handler, resolver request.RequestInfoResolver) http.Handler {
+	return withPanicRecovery(handler, func(w http.ResponseWriter, req *http.Request, err interface{}) {
+		if err == http.ErrAbortHandler {
+			// Honor the http.ErrAbortHandler sentinel panic value
+			//
+			// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+			// that the effect of the panic was isolated to the active request.
+			// It recovers the panic, logs a stack trace to the server error log,
+			// and either closes the network connection or sends an HTTP/2
+			// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+			// the client sees an interrupted response but the server doesn't log
+			// an error, panic with the value ErrAbortHandler.
+			//
+			// Note that HandleCrash function is actually crashing, after calling the handlers
+			if info, err := resolver.NewRequestInfo(req); err != nil {
+				metrics.RecordRequestAbort(req, nil)
+			} else {
+				metrics.RecordRequestAbort(req, info)
+			}
+			// This call can have different handlers, but the default chain rate limits. Call it after the metrics are updated
+			// in case the rate limit delays it.  If you outrun the rate for this one timed out requests, something has gone
+			// seriously wrong with your server, but generally having a logging signal for timeouts is useful.
+			runtime.HandleErrorWithContext(req.Context(), nil, "Timeout or abort while handling", "method", req.Method, "URI", req.RequestURI, "auditID", audit.GetAuditIDTruncated(req.Context()))
+			return
+		}
+		http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
+		klog.ErrorS(nil, "apiserver panic'd", "method", req.Method, "URI", req.RequestURI, "auditID", audit.GetAuditIDTruncated(req.Context()))
+	})
+}
+
+// WithHTTPLogging enables logging of incoming requests.
+func WithHTTPLogging(handler http.Handler) http.Handler {
+	return httplog.WithLogging(handler, httplog.DefaultStacktracePred)
+}
+
+func withPanicRecovery(handler http.Handler, crashHandler func(http.ResponseWriter, *http.Request, interface{})) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer runtime.HandleCrash(func(err interface{}) {
+			crashHandler(w, req, err)
+		})
+
+		// Dispatch to the internal handler
+		handler.ServeHTTP(w, req)
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/handler.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	rt "runtime"
+	"sort"
+	"strings"
+
+	"github.com/emicklei/go-restful/v3"
+	"k8s.io/klog/v2"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// APIServerHandlers holds the different http.Handlers used by the API server.
+// This includes the full handler chain, the director (which chooses between gorestful and nonGoRestful,
+// the gorestful handler (used for the API) which falls through to the nonGoRestful handler on unregistered paths,
+// and the nonGoRestful handler (which can contain a fallthrough of its own)
+// FullHandlerChain -> Director -> {GoRestfulContainer,NonGoRestfulMux} based on inspection of registered web services
+type APIServerHandler struct {
+	// FullHandlerChain is the one that is eventually served with.  It should include the full filter
+	// chain and then call the Director.
+	FullHandlerChain http.Handler
+	// The registered APIs.  InstallAPIs uses this.  Other servers probably shouldn't access this directly.
+	GoRestfulContainer *restful.Container
+	// NonGoRestfulMux is the final HTTP handler in the chain.
+	// It comes after all filters and the API handling
+	// This is where other servers can attach handler to various parts of the chain.
+	NonGoRestfulMux *mux.PathRecorderMux
+
+	// Director is here so that we can properly handle fall through and proxy cases.
+	// This looks a bit bonkers, but here's what's happening.  We need to have /apis handling registered in gorestful in order to have
+	// swagger generated for compatibility.  Doing that with `/apis` as a webservice, means that it forcibly 404s (no defaulting allowed)
+	// all requests which are not /apis or /apis/.  We need those calls to fall through behind gorestful for proper delegation.  Trying to
+	// register for a pattern which includes everything behind it doesn't work because gorestful negotiates for verbs and content encoding
+	// and all those things go crazy when gorestful really just needs to pass through.  In addition, openapi enforces unique verb constraints
+	// which we don't fit into and it still muddies up swagger.  Trying to switch the webservices into a route doesn't work because the
+	//  containing webservice faces all the same problems listed above.
+	// This leads to the crazy thing done here.  Our mux does what we need, so we'll place it in front of gorestful.  It will introspect to
+	// decide if the route is likely to be handled by gorestful and route there if needed.  Otherwise, it goes to NonGoRestfulMux mux in
+	// order to handle "normal" paths and delegation. Hopefully no API consumers will ever have to deal with this level of detail.  I think
+	// we should consider completely removing gorestful.
+	// Other servers should only use this opaquely to delegate to an API server.
+	Director http.Handler
+}
+
+// HandlerChainBuilderFn is used to wrap the GoRestfulContainer handler using the provided handler chain.
+// It is normally used to apply filtering like authentication and authorization
+type HandlerChainBuilderFn func(apiHandler http.Handler) http.Handler
+
+func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
+	nonGoRestfulMux := mux.NewPathRecorderMux(name)
+	if notFoundHandler != nil {
+		nonGoRestfulMux.NotFoundHandler(notFoundHandler)
+	}
+
+	gorestfulContainer := restful.NewContainer()
+	gorestfulContainer.Router(restful.CurlyRouter{}) // e.g. for proxy/{kind}/{name}/{*}
+	gorestfulContainer.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
+		logStackOnRecover(s, panicReason, httpWriter)
+	})
+	gorestfulContainer.ServiceErrorHandler(func(serviceErr restful.ServiceError, request *restful.Request, response *restful.Response) {
+		serviceErrorHandler(s, serviceErr, request, response)
+	})
+
+	director := director{
+		name:               name,
+		goRestfulContainer: gorestfulContainer,
+		nonGoRestfulMux:    nonGoRestfulMux,
+	}
+
+	return &APIServerHandler{
+		FullHandlerChain:   handlerChainBuilder(director),
+		GoRestfulContainer: gorestfulContainer,
+		NonGoRestfulMux:    nonGoRestfulMux,
+		Director:           director,
+	}
+}
+
+// ListedPaths returns the paths that should be shown under /
+func (a *APIServerHandler) ListedPaths() []string {
+	var handledPaths []string
+	// Extract the paths handled using restful.WebService
+	for _, ws := range a.GoRestfulContainer.RegisteredWebServices() {
+		handledPaths = append(handledPaths, ws.RootPath())
+	}
+	handledPaths = append(handledPaths, a.NonGoRestfulMux.ListedPaths()...)
+	sort.Strings(handledPaths)
+
+	return handledPaths
+}
+
+type director struct {
+	name               string
+	goRestfulContainer *restful.Container
+	nonGoRestfulMux    *mux.PathRecorderMux
+}
+
+func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	path := req.URL.Path
+
+	// check to see if our webservices want to claim this path
+	for _, ws := range d.goRestfulContainer.RegisteredWebServices() {
+		switch {
+		case ws.RootPath() == "/apis":
+			// if we are exactly /apis or /apis/, then we need special handling in loop.
+			// normally these are passed to the nonGoRestfulMux, but if discovery is enabled, it will go directly.
+			// We can't rely on a prefix match since /apis matches everything (see the big comment on Director above)
+			if path == "/apis" || path == "/apis/" {
+				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
+				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
+				// TODO fix gorestful, remove TPRs, or stop using gorestful
+				d.goRestfulContainer.Dispatch(w, req)
+				return
+			}
+
+		case strings.HasPrefix(path, ws.RootPath()):
+			// ensure an exact match or a path boundary match
+			if len(path) == len(ws.RootPath()) || path[len(ws.RootPath())] == '/' {
+				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
+				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
+				// TODO fix gorestful, remove TPRs, or stop using gorestful
+				d.goRestfulContainer.Dispatch(w, req)
+				return
+			}
+		}
+	}
+
+	// if we didn't find a match, then we just skip gorestful altogether
+	klog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path)
+	d.nonGoRestfulMux.ServeHTTP(w, req)
+}
+
+// TODO: Unify with RecoverPanics?
+func logStackOnRecover(s runtime.NegotiatedSerializer, panicReason interface{}, w http.ResponseWriter) {
+	var buffer bytes.Buffer
+	buffer.WriteString(fmt.Sprintf("recover from panic situation: - %v\r\n", panicReason))
+	for i := 2; ; i++ {
+		_, file, line, ok := rt.Caller(i)
+		if !ok {
+			break
+		}
+		buffer.WriteString(fmt.Sprintf("    %s:%d\r\n", file, line))
+	}
+	klog.Errorln(buffer.String())
+
+	headers := http.Header{}
+	if ct := w.Header().Get("Content-Type"); len(ct) > 0 {
+		headers.Set("Accept", ct)
+	}
+	responsewriters.ErrorNegotiated(apierrors.NewGenericServerResponse(http.StatusInternalServerError, "", schema.GroupResource{}, "", "", 0, false), s, schema.GroupVersion{}, w, &http.Request{Header: headers})
+}
+
+func serviceErrorHandler(s runtime.NegotiatedSerializer, serviceErr restful.ServiceError, request *restful.Request, resp *restful.Response) {
+	responsewriters.ErrorNegotiated(
+		apierrors.NewGenericServerResponse(serviceErr.Code, "", schema.GroupResource{}, "", serviceErr.Message, 0, false),
+		s,
+		schema.GroupVersion{},
+		resp,
+		request.Request,
+	)
+}
+
+// ServeHTTP makes it an http.Handler
+func (a *APIServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	a.FullHandlerChain.ServeHTTP(w, r)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/healthz.go

@@ -0,0 +1,167 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/utils/clock"
+)
+
+// healthMux is an interface describing the methods InstallHandler requires.
+type healthMux interface {
+	Handle(pattern string, handler http.Handler)
+}
+
+type healthCheckRegistry struct {
+	path            string
+	lock            sync.Mutex
+	checks          []healthz.HealthChecker
+	checksInstalled bool
+	clock           clock.Clock
+}
+
+func (reg *healthCheckRegistry) addHealthChecks(checks ...healthz.HealthChecker) error {
+	return reg.addDelayedHealthChecks(0, checks...)
+}
+
+func (reg *healthCheckRegistry) addDelayedHealthChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
+	if delay > 0 && reg.clock == nil {
+		return fmt.Errorf("nil clock in healthCheckRegistry for %s endpoint", reg.path)
+	}
+	reg.lock.Lock()
+	defer reg.lock.Unlock()
+	if reg.checksInstalled {
+		return fmt.Errorf("unable to add because the %s endpoint has already been created", reg.path)
+	}
+	if delay > 0 {
+		for _, check := range checks {
+			reg.checks = append(reg.checks, delayedHealthCheck(check, reg.clock, delay))
+		}
+	} else {
+		reg.checks = append(reg.checks, checks...)
+	}
+	return nil
+}
+
+func (reg *healthCheckRegistry) installHandler(mux healthMux) {
+	reg.installHandlerWithHealthyFunc(mux, nil)
+}
+
+func (reg *healthCheckRegistry) installHandlerWithHealthyFunc(mux healthMux, firstTimeHealthy func()) {
+	reg.lock.Lock()
+	defer reg.lock.Unlock()
+	reg.checksInstalled = true
+	healthz.InstallPathHandlerWithHealthyFunc(mux, reg.path, firstTimeHealthy, reg.checks...)
+}
+
+// AddHealthChecks adds HealthCheck(s) to health endpoints (healthz, livez, readyz) but
+// configures the liveness grace period to be zero, which means we expect this health check
+// to immediately indicate that the apiserver is unhealthy.
+func (s *GenericAPIServer) AddHealthChecks(checks ...healthz.HealthChecker) error {
+	// we opt for a delay of zero here, because this entrypoint adds generic health checks
+	// and not health checks which are specifically related to kube-apiserver boot-sequences.
+	return s.addHealthChecks(0, checks...)
+}
+
+// AddBootSequenceHealthChecks adds health checks to the old healthz endpoint (for backwards compatibility reasons)
+// as well as livez and readyz. The livez grace period is defined by the value of the
+// command-line flag --livez-grace-period; before the grace period elapses, the livez health checks
+// will default to healthy. One may want to set a grace period in order to prevent the kubelet from restarting
+// the kube-apiserver due to long-ish boot sequences. Readyz health checks, on the other hand, have no grace period,
+// since readyz should fail until boot fully completes.
+func (s *GenericAPIServer) AddBootSequenceHealthChecks(checks ...healthz.HealthChecker) error {
+	return s.addHealthChecks(s.livezGracePeriod, checks...)
+}
+
+// addHealthChecks adds health checks to healthz, livez, and readyz. The delay passed in will set
+// a corresponding grace period on livez.
+func (s *GenericAPIServer) addHealthChecks(livezGracePeriod time.Duration, checks ...healthz.HealthChecker) error {
+	if err := s.healthzRegistry.addHealthChecks(checks...); err != nil {
+		return err
+	}
+	if err := s.livezRegistry.addDelayedHealthChecks(livezGracePeriod, checks...); err != nil {
+		return err
+	}
+	return s.readyzRegistry.addHealthChecks(checks...)
+}
+
+// AddReadyzChecks allows you to add a HealthCheck to readyz.
+func (s *GenericAPIServer) AddReadyzChecks(checks ...healthz.HealthChecker) error {
+	return s.readyzRegistry.addHealthChecks(checks...)
+}
+
+// AddLivezChecks allows you to add a HealthCheck to livez.
+func (s *GenericAPIServer) AddLivezChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
+	return s.livezRegistry.addDelayedHealthChecks(delay, checks...)
+}
+
+// addReadyzShutdownCheck is a convenience function for adding a readyz shutdown check, so
+// that we can register that the api-server is no longer ready while we attempt to gracefully
+// shutdown.
+func (s *GenericAPIServer) addReadyzShutdownCheck(stopCh <-chan struct{}) error {
+	return s.AddReadyzChecks(healthz.NewShutdownHealthz(stopCh))
+}
+
+// installHealthz creates the healthz endpoint for this server
+func (s *GenericAPIServer) installHealthz() {
+	s.healthzRegistry.installHandler(s.Handler.NonGoRestfulMux)
+}
+
+// installReadyz creates the readyz endpoint for this server.
+func (s *GenericAPIServer) installReadyz() {
+	s.readyzRegistry.installHandlerWithHealthyFunc(s.Handler.NonGoRestfulMux, func() {
+		// note: installHandlerWithHealthyFunc guarantees that this is called only once
+		s.lifecycleSignals.HasBeenReady.Signal()
+	})
+}
+
+// installLivez creates the livez endpoint for this server.
+func (s *GenericAPIServer) installLivez() {
+	s.livezRegistry.installHandler(s.Handler.NonGoRestfulMux)
+}
+
+// delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed. This
+// is intended for use primarily for livez health checks.
+func delayedHealthCheck(check healthz.HealthChecker, clock clock.Clock, delay time.Duration) healthz.HealthChecker {
+	return delayedLivezCheck{
+		check,
+		clock.Now().Add(delay),
+		clock,
+	}
+}
+
+type delayedLivezCheck struct {
+	check      healthz.HealthChecker
+	startCheck time.Time
+	clock      clock.Clock
+}
+
+func (c delayedLivezCheck) Name() string {
+	return c.check.Name()
+}
+
+func (c delayedLivezCheck) Check(req *http.Request) error {
+	if c.clock.Now().After(c.startCheck) {
+		return c.check.Check(req)
+	}
+	return nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/healthz/doc.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package healthz implements basic http server health checking.
+// Usage:
+//
+//	import "k8s.io/apiserver/pkg/server/healthz"
+//	healthz.InstallHandler(mux)
+package healthz // import "k8s.io/apiserver/pkg/server/healthz"

e2e/vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -0,0 +1,333 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package healthz
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/component-base/metrics/prometheus/slis"
+	"k8s.io/klog/v2"
+)
+
+const DefaultHealthzPath = "/healthz"
+
+// HealthChecker is a named healthz checker.
+type HealthChecker interface {
+	Name() string
+	Check(req *http.Request) error
+}
+
+// PingHealthz returns true automatically when checked
+var PingHealthz HealthChecker = ping{}
+
+// ping implements the simplest possible healthz checker.
+type ping struct{}
+
+func (ping) Name() string {
+	return "ping"
+}
+
+// PingHealthz is a health check that returns true.
+func (ping) Check(_ *http.Request) error {
+	return nil
+}
+
+// LogHealthz returns true if logging is not blocked
+var LogHealthz HealthChecker = &log{}
+
+type log struct {
+	startOnce    sync.Once
+	lastVerified atomic.Value
+}
+
+func (l *log) Name() string {
+	return "log"
+}
+
+func (l *log) Check(_ *http.Request) error {
+	l.startOnce.Do(func() {
+		l.lastVerified.Store(time.Now())
+		go wait.Forever(func() {
+			klog.Flush()
+			l.lastVerified.Store(time.Now())
+		}, time.Minute)
+	})
+
+	lastVerified := l.lastVerified.Load().(time.Time)
+	if time.Since(lastVerified) < (2 * time.Minute) {
+		return nil
+	}
+	return fmt.Errorf("logging blocked")
+}
+
+type cacheSyncWaiter interface {
+	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
+}
+
+type informerSync struct {
+	cacheSyncWaiter cacheSyncWaiter
+}
+
+var _ HealthChecker = &informerSync{}
+
+// NewInformerSyncHealthz returns a new HealthChecker that will pass only if all informers in the given cacheSyncWaiter sync.
+func NewInformerSyncHealthz(cacheSyncWaiter cacheSyncWaiter) HealthChecker {
+	return &informerSync{
+		cacheSyncWaiter: cacheSyncWaiter,
+	}
+}
+
+func (i *informerSync) Name() string {
+	return "informer-sync"
+}
+
+type shutdown struct {
+	stopCh <-chan struct{}
+}
+
+// NewShutdownHealthz returns a new HealthChecker that will fail if the embedded channel is closed.
+// This is intended to allow for graceful shutdown sequences.
+func NewShutdownHealthz(stopCh <-chan struct{}) HealthChecker {
+	return &shutdown{stopCh}
+}
+
+func (s *shutdown) Name() string {
+	return "shutdown"
+}
+
+func (s *shutdown) Check(req *http.Request) error {
+	select {
+	case <-s.stopCh:
+		return fmt.Errorf("process is shutting down")
+	default:
+	}
+	return nil
+}
+
+func (i *informerSync) Check(_ *http.Request) error {
+	stopCh := make(chan struct{})
+	// Close stopCh to force checking if informers are synced now.
+	close(stopCh)
+
+	informersByStarted := make(map[bool][]string)
+	for informerType, started := range i.cacheSyncWaiter.WaitForCacheSync(stopCh) {
+		informersByStarted[started] = append(informersByStarted[started], informerType.String())
+	}
+
+	if notStarted := informersByStarted[false]; len(notStarted) > 0 {
+		return fmt.Errorf("%d informers not started yet: %v", len(notStarted), notStarted)
+	}
+	return nil
+}
+
+// NamedCheck returns a healthz checker for the given name and function.
+func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
+	return &healthzCheck{name, check}
+}
+
+// InstallHandler registers handlers for health checking on the path
+// "/healthz" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, DefaultHealthzPath, checks...)
+}
+
+// InstallReadyzHandler registers handlers for health checking on the path
+// "/readyz" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/readyz", checks...)
+}
+
+// InstallLivezHandler registers handlers for liveness checking on the path
+// "/livez" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallLivezHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/livez", checks...)
+}
+
+// InstallPathHandler registers handlers for health checking on
+// a specific path to mux. *All handlers* for the path must be
+// specified in exactly one call to InstallPathHandler. Calling
+// InstallPathHandler more than once for the same path and mux will
+// result in a panic.
+func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
+	InstallPathHandlerWithHealthyFunc(mux, path, nil, checks...)
+}
+
+// InstallPathHandlerWithHealthyFunc is like InstallPathHandler, but calls firstTimeHealthy exactly once
+// when the handler succeeds for the first time.
+func InstallPathHandlerWithHealthyFunc(mux mux, path string, firstTimeHealthy func(), checks ...HealthChecker) {
+	if len(checks) == 0 {
+		klog.V(5).Info("No default health checks specified. Installing the ping handler.")
+		checks = []HealthChecker{PingHealthz}
+	}
+
+	klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
+
+	name := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
+	mux.Handle(path,
+		metrics.InstrumentHandlerFunc("GET",
+			/* group = */ "",
+			/* version = */ "",
+			/* resource = */ "",
+			/* subresource = */ path,
+			/* scope = */ "",
+			/* component = */ "",
+			/* deprecated */ false,
+			/* removedRelease */ "",
+			handleRootHealth(name, firstTimeHealthy, checks...)))
+	for _, check := range checks {
+		mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
+	}
+}
+
+// mux is an interface describing the methods InstallHandler requires.
+type mux interface {
+	Handle(pattern string, handler http.Handler)
+}
+
+// healthzCheck implements HealthChecker on an arbitrary name and check function.
+type healthzCheck struct {
+	name  string
+	check func(r *http.Request) error
+}
+
+var _ HealthChecker = &healthzCheck{}
+
+func (c *healthzCheck) Name() string {
+	return c.name
+}
+
+func (c *healthzCheck) Check(r *http.Request) error {
+	return c.check(r)
+}
+
+// getExcludedChecks extracts the health check names to be excluded from the query param
+func getExcludedChecks(r *http.Request) sets.String {
+	checks, found := r.URL.Query()["exclude"]
+	if found {
+		return sets.NewString(checks...)
+	}
+	return sets.NewString()
+}
+
+// handleRootHealth returns an http.HandlerFunc that serves the provided checks.
+func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChecker) http.HandlerFunc {
+	var notifyOnce sync.Once
+	return func(w http.ResponseWriter, r *http.Request) {
+		excluded := getExcludedChecks(r)
+		// failedVerboseLogOutput is for output to the log.  It indicates detailed failed output information for the log.
+		var failedVerboseLogOutput bytes.Buffer
+		var failedChecks []string
+		var individualCheckOutput bytes.Buffer
+		for _, check := range checks {
+			// no-op the check if we've specified we want to exclude the check
+			if excluded.Has(check.Name()) {
+				excluded.Delete(check.Name())
+				fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
+				continue
+			}
+			if err := check.Check(r); err != nil {
+				slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Error)
+				// don't include the error since this endpoint is public.  If someone wants more detail
+				// they should have explicit permission to the detailed checks.
+				fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
+				// but we do want detailed information for our log
+				fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
+				failedChecks = append(failedChecks, check.Name())
+			} else {
+				slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Success)
+				fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
+			}
+		}
+		if excluded.Len() > 0 {
+			fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
+			klog.V(6).Infof("cannot exclude some health checks, no health checks are installed matching %s",
+				formatQuoted(excluded.List()...))
+		}
+		// always be verbose on failure
+		if len(failedChecks) > 0 {
+			klog.V(2).Infof("%s check failed: %s\n%v", strings.Join(failedChecks, ","), name, failedVerboseLogOutput.String())
+			httplog.SetStacktracePredicate(r.Context(), func(int) bool { return false })
+			http.Error(w, fmt.Sprintf("%s%s check failed", individualCheckOutput.String(), name), http.StatusInternalServerError)
+			return
+		}
+
+		// signal first time this is healthy
+		if firstTimeHealthy != nil {
+			notifyOnce.Do(firstTimeHealthy)
+		}
+
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		if _, found := r.URL.Query()["verbose"]; !found {
+			fmt.Fprint(w, "ok")
+			return
+		}
+
+		individualCheckOutput.WriteTo(w)
+		fmt.Fprintf(w, "%s check passed\n", name)
+	}
+}
+
+// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
+func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := c(r)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
+		} else {
+			fmt.Fprint(w, "ok")
+		}
+	})
+}
+
+// checkerNames returns the names of the checks in the same order as passed in.
+func checkerNames(checks ...HealthChecker) []string {
+	// accumulate the names of checks for printing them out.
+	checkerNames := make([]string, 0, len(checks))
+	for _, check := range checks {
+		checkerNames = append(checkerNames, check.Name())
+	}
+	return checkerNames
+}
+
+// formatQuoted returns a formatted string of the health check names,
+// preserving the order passed in.
+func formatQuoted(names ...string) string {
+	quoted := make([]string, 0, len(names))
+	for _, name := range names {
+		quoted = append(quoted, fmt.Sprintf("%q", name))
+	}
+	return strings.Join(quoted, ",")
+}

e2e/vendor/k8s.io/apiserver/pkg/server/hooks.go

@@ -0,0 +1,246 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"runtime/debug"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/server/healthz"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+)
+
+// PostStartHookFunc is a function that is called after the server has started.
+// It must properly handle cases like:
+//  1. asynchronous start in multiple API server processes
+//  2. conflicts between the different processes all trying to perform the same action
+//  3. partially complete work (API server crashes while running your hook)
+//  4. API server access **BEFORE** your hook has completed
+//
+// Think of it like a mini-controller that is super privileged and gets to run in-process
+// If you use this feature, tag @deads2k on github who has promised to review code for anyone's PostStartHook
+// until it becomes easier to use.
+type PostStartHookFunc func(context PostStartHookContext) error
+
+// PreShutdownHookFunc is a function that can be added to the shutdown logic.
+type PreShutdownHookFunc func() error
+
+// PostStartHookContext provides information about this API server to a PostStartHookFunc
+type PostStartHookContext struct {
+	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
+	LoopbackClientConfig *restclient.Config
+	// Context gets cancelled when the server stops.
+	context.Context
+}
+
+// PostStartHookProvider is an interface in addition to provide a post start hook for the api server
+type PostStartHookProvider interface {
+	PostStartHook() (string, PostStartHookFunc, error)
+}
+
+type postStartHookEntry struct {
+	hook PostStartHookFunc
+	// originatingStack holds the stack that registered postStartHooks. This allows us to show a more helpful message
+	// for duplicate registration.
+	originatingStack string
+
+	// done will be closed when the postHook is finished
+	done chan struct{}
+}
+
+type PostStartHookConfigEntry struct {
+	hook PostStartHookFunc
+	// originatingStack holds the stack that registered postStartHooks. This allows us to show a more helpful message
+	// for duplicate registration.
+	originatingStack string
+}
+
+type preShutdownHookEntry struct {
+	hook PreShutdownHookFunc
+}
+
+// AddPostStartHook allows you to add a PostStartHook.
+func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc) error {
+	if len(name) == 0 {
+		return fmt.Errorf("missing name")
+	}
+	if hook == nil {
+		return fmt.Errorf("hook func may not be nil: %q", name)
+	}
+	if s.disabledPostStartHooks.Has(name) {
+		klog.V(1).Infof("skipping %q because it was explicitly disabled", name)
+		return nil
+	}
+
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+
+	if s.postStartHooksCalled {
+		return fmt.Errorf("unable to add %q because PostStartHooks have already been called", name)
+	}
+	if postStartHook, exists := s.postStartHooks[name]; exists {
+		// this is programmer error, but it can be hard to debug
+		return fmt.Errorf("unable to add %q because it was already registered by: %s", name, postStartHook.originatingStack)
+	}
+
+	// done is closed when the poststarthook is finished.  This is used by the health check to be able to indicate
+	// that the poststarthook is finished
+	done := make(chan struct{})
+	if err := s.AddBootSequenceHealthChecks(postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
+		return err
+	}
+	s.postStartHooks[name] = postStartHookEntry{hook: hook, originatingStack: string(debug.Stack()), done: done}
+
+	return nil
+}
+
+// AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure
+func (s *GenericAPIServer) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
+	if err := s.AddPostStartHook(name, hook); err != nil {
+		klog.Fatalf("Error registering PostStartHook %q: %v", name, err)
+	}
+}
+
+// AddPreShutdownHook allows you to add a PreShutdownHook.
+func (s *GenericAPIServer) AddPreShutdownHook(name string, hook PreShutdownHookFunc) error {
+	if len(name) == 0 {
+		return fmt.Errorf("missing name")
+	}
+	if hook == nil {
+		return nil
+	}
+
+	s.preShutdownHookLock.Lock()
+	defer s.preShutdownHookLock.Unlock()
+
+	if s.preShutdownHooksCalled {
+		return fmt.Errorf("unable to add %q because PreShutdownHooks have already been called", name)
+	}
+	if _, exists := s.preShutdownHooks[name]; exists {
+		return fmt.Errorf("unable to add %q because it is already registered", name)
+	}
+
+	s.preShutdownHooks[name] = preShutdownHookEntry{hook: hook}
+
+	return nil
+}
+
+// AddPreShutdownHookOrDie allows you to add a PostStartHook, but dies on failure
+func (s *GenericAPIServer) AddPreShutdownHookOrDie(name string, hook PreShutdownHookFunc) {
+	if err := s.AddPreShutdownHook(name, hook); err != nil {
+		klog.Fatalf("Error registering PreShutdownHook %q: %v", name, err)
+	}
+}
+
+// RunPostStartHooks runs the PostStartHooks for the server.
+func (s *GenericAPIServer) RunPostStartHooks(ctx context.Context) {
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+	s.postStartHooksCalled = true
+
+	context := PostStartHookContext{
+		LoopbackClientConfig: s.LoopbackClientConfig,
+		Context:              ctx,
+	}
+
+	for hookName, hookEntry := range s.postStartHooks {
+		go runPostStartHook(hookName, hookEntry, context)
+	}
+}
+
+// RunPreShutdownHooks runs the PreShutdownHooks for the server
+func (s *GenericAPIServer) RunPreShutdownHooks() error {
+	var errorList []error
+
+	s.preShutdownHookLock.Lock()
+	defer s.preShutdownHookLock.Unlock()
+	s.preShutdownHooksCalled = true
+
+	for hookName, hookEntry := range s.preShutdownHooks {
+		if err := runPreShutdownHook(hookName, hookEntry); err != nil {
+			errorList = append(errorList, err)
+		}
+	}
+	return utilerrors.NewAggregate(errorList)
+}
+
+// isPostStartHookRegistered checks whether a given PostStartHook is registered
+func (s *GenericAPIServer) isPostStartHookRegistered(name string) bool {
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+	_, exists := s.postStartHooks[name]
+	return exists
+}
+
+func runPostStartHook(name string, entry postStartHookEntry, context PostStartHookContext) {
+	var err error
+	func() {
+		// don't let the hook *accidentally* panic and kill the server
+		defer utilruntime.HandleCrash()
+		err = entry.hook(context)
+	}()
+	// if the hook intentionally wants to kill server, let it.
+	if err != nil {
+		klog.Fatalf("PostStartHook %q failed: %v", name, err)
+	}
+	close(entry.done)
+}
+
+func runPreShutdownHook(name string, entry preShutdownHookEntry) error {
+	var err error
+	func() {
+		// don't let the hook *accidentally* panic and kill the server
+		defer utilruntime.HandleCrash()
+		err = entry.hook()
+	}()
+	if err != nil {
+		return fmt.Errorf("PreShutdownHook %q failed: %v", name, err)
+	}
+	return nil
+}
+
+// postStartHookHealthz implements a healthz check for poststarthooks.  It will return a "hookNotFinished"
+// error until the poststarthook is finished.
+type postStartHookHealthz struct {
+	name string
+
+	// done will be closed when the postStartHook is finished
+	done chan struct{}
+}
+
+var _ healthz.HealthChecker = postStartHookHealthz{}
+
+func (h postStartHookHealthz) Name() string {
+	return h.name
+}
+
+var errHookNotFinished = errors.New("not finished")
+
+func (h postStartHookHealthz) Check(req *http.Request) error {
+	select {
+	case <-h.done:
+		return nil
+	default:
+		return errHookNotFinished
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/httplog/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package httplog contains a helper object and functions to maintain a log
+// along with an http response.
+package httplog // import "k8s.io/apiserver/pkg/server/httplog"

e2e/vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -0,0 +1,342 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package httplog
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/server/routine"
+	"k8s.io/klog/v2"
+)
+
+// StacktracePred returns true if a stacktrace should be logged for this status.
+type StacktracePred func(httpStatus int) (logStacktrace bool)
+
+// ShouldLogRequestPred returns true if logging should be enabled for this request
+type ShouldLogRequestPred func() bool
+
+type logger interface {
+	Addf(format string, data ...interface{})
+}
+
+type respLoggerContextKeyType int
+
+// respLoggerContextKey is used to store the respLogger pointer in the request context.
+const respLoggerContextKey respLoggerContextKeyType = iota
+
+// Add a layer on top of ResponseWriter, so we can track latency and error
+// message sources.
+//
+// TODO now that we're using go-restful, we shouldn't need to be wrapping
+// the http.ResponseWriter. We can recover panics from go-restful, and
+// the logging value is questionable.
+type respLogger struct {
+	hijacked       bool
+	statusRecorded bool
+	status         int
+	statusStack    string
+	// mutex is used when accessing addedInfo, addedKeyValuePairs and logStacktracePred.
+	// They can be modified by other goroutine when logging happens (in case of request timeout)
+	mutex              sync.Mutex
+	addedInfo          strings.Builder
+	addedKeyValuePairs []interface{}
+	startTime          time.Time
+
+	captureErrorOutput bool
+
+	req       *http.Request
+	userAgent string
+	w         http.ResponseWriter
+
+	logStacktracePred StacktracePred
+}
+
+var _ http.ResponseWriter = &respLogger{}
+var _ responsewriter.UserProvidedDecorator = &respLogger{}
+
+func (rl *respLogger) Unwrap() http.ResponseWriter {
+	return rl.w
+}
+
+// Simple logger that logs immediately when Addf is called
+type passthroughLogger struct{}
+
+// Addf logs info immediately.
+func (passthroughLogger) Addf(format string, data ...interface{}) {
+	klog.V(2).Info(fmt.Sprintf(format, data...))
+}
+
+// DefaultStacktracePred is the default implementation of StacktracePred.
+func DefaultStacktracePred(status int) bool {
+	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
+}
+
+const withLoggingLevel = 3
+
+// WithLogging wraps the handler with logging.
+func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
+	return withLogging(handler, pred, func() bool {
+		return klog.V(withLoggingLevel).Enabled()
+	})
+}
+
+func withLogging(handler http.Handler, stackTracePred StacktracePred, shouldLogRequest ShouldLogRequestPred) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if !shouldLogRequest() {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		ctx := req.Context()
+		if old := respLoggerFromRequest(req); old != nil {
+			panic("multiple WithLogging calls!")
+		}
+
+		startTime := time.Now()
+		if receivedTimestamp, ok := request.ReceivedTimestampFrom(ctx); ok {
+			startTime = receivedTimestamp
+		}
+
+		rl := newLoggedWithStartTime(req, w, startTime)
+		rl.StacktraceWhen(stackTracePred)
+		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
+
+		var logFunc func()
+		logFunc = rl.Log
+		defer func() {
+			if logFunc != nil {
+				logFunc()
+			}
+		}()
+
+		w = responsewriter.WrapForHTTP1Or2(rl)
+		handler.ServeHTTP(w, req)
+
+		// We need to ensure that the request is logged after it is processed.
+		// In case the request is executed in a separate goroutine created via
+		// WithRoutine handler in the handler chain (i.e. above handler.ServeHTTP()
+		// would return request is completely responsed), we want the logging to
+		// happen in that goroutine too, so we append it to the task.
+		if routine.AppendTask(ctx, &routine.Task{Func: rl.Log}) {
+			logFunc = nil
+		}
+	})
+}
+
+// respLoggerFromContext returns the respLogger or nil.
+func respLoggerFromContext(ctx context.Context) *respLogger {
+	val := ctx.Value(respLoggerContextKey)
+	if rl, ok := val.(*respLogger); ok {
+		return rl
+	}
+	return nil
+}
+
+func respLoggerFromRequest(req *http.Request) *respLogger {
+	return respLoggerFromContext(req.Context())
+}
+
+func newLoggedWithStartTime(req *http.Request, w http.ResponseWriter, startTime time.Time) *respLogger {
+	logger := &respLogger{
+		startTime:         startTime,
+		req:               req,
+		userAgent:         req.UserAgent(),
+		w:                 w,
+		logStacktracePred: DefaultStacktracePred,
+	}
+	return logger
+}
+
+// newLogged turns a normal response writer into a logged response writer.
+func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
+	return newLoggedWithStartTime(req, w, time.Now())
+}
+
+// LogOf returns the logger hiding in w. If there is not an existing logger
+// then a passthroughLogger will be created which will log to stdout immediately
+// when Addf is called.
+func LogOf(req *http.Request, w http.ResponseWriter) logger {
+	if rl := respLoggerFromRequest(req); rl != nil {
+		return rl
+	}
+	return &passthroughLogger{}
+}
+
+// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
+func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
+	if rl := respLoggerFromRequest(req); rl != nil {
+		return rl.w
+	}
+	return w
+}
+
+// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
+// There's a default, so you don't need to call this unless you don't like the default.
+func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.logStacktracePred = pred
+	return rl
+}
+
+// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
+// for any status *not* in the given list.
+func StatusIsNot(statuses ...int) StacktracePred {
+	statusesNoTrace := map[int]bool{}
+	for _, s := range statuses {
+		statusesNoTrace[s] = true
+	}
+	return func(status int) bool {
+		_, ok := statusesNoTrace[status]
+		return !ok
+	}
+}
+
+// Addf adds additional data to be logged with this request.
+func (rl *respLogger) Addf(format string, data ...interface{}) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.addedInfo.WriteString(fmt.Sprintf(format, data...))
+}
+
+func AddInfof(ctx context.Context, format string, data ...interface{}) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.Addf(format, data...)
+	}
+}
+
+func (rl *respLogger) AddKeyValue(key string, value interface{}) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.addedKeyValuePairs = append(rl.addedKeyValuePairs, key, value)
+}
+
+// AddKeyValue adds a (key, value) pair to the httplog associated
+// with the request.
+// Use this function if you want your data to show up in httplog
+// in a more structured and readable way.
+func AddKeyValue(ctx context.Context, key string, value interface{}) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.AddKeyValue(key, value)
+	}
+}
+
+// SetStacktracePredicate sets a custom stacktrace predicate for the
+// logger associated with the given request context.
+func SetStacktracePredicate(ctx context.Context, pred StacktracePred) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.StacktraceWhen(pred)
+	}
+}
+
+// Log is intended to be called once at the end of your request handler, via defer
+func (rl *respLogger) Log() {
+	latency := time.Since(rl.startTime)
+	auditID := audit.GetAuditIDTruncated(rl.req.Context())
+	verb := metrics.NormalizedVerb(rl.req)
+
+	keysAndValues := []interface{}{
+		"verb", verb,
+		"URI", rl.req.RequestURI,
+		"latency", latency,
+		// We can't get UserAgent from rl.req.UserAgent() here as it accesses headers map,
+		// which can be modified in another goroutine when apiserver request times out.
+		// For example authentication filter modifies request's headers,
+		// This can cause apiserver to crash with unrecoverable fatal error.
+		// More info about concurrent read and write for maps: https://golang.org/doc/go1.6#runtime
+		"userAgent", rl.userAgent,
+		"audit-ID", auditID,
+		"srcIP", rl.req.RemoteAddr,
+	}
+	// Lock for accessing addedKeyValuePairs and addedInfo
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	keysAndValues = append(keysAndValues, rl.addedKeyValuePairs...)
+
+	if rl.hijacked {
+		keysAndValues = append(keysAndValues, "hijacked", true)
+	} else {
+		keysAndValues = append(keysAndValues, "resp", rl.status)
+		if len(rl.statusStack) > 0 {
+			keysAndValues = append(keysAndValues, "statusStack", rl.statusStack)
+		}
+		info := rl.addedInfo.String()
+		if len(info) > 0 {
+			keysAndValues = append(keysAndValues, "addedInfo", info)
+		}
+	}
+
+	klog.V(withLoggingLevel).InfoSDepth(1, "HTTP", keysAndValues...)
+}
+
+// Header implements http.ResponseWriter.
+func (rl *respLogger) Header() http.Header {
+	return rl.w.Header()
+}
+
+// Write implements http.ResponseWriter.
+func (rl *respLogger) Write(b []byte) (int, error) {
+	if !rl.statusRecorded {
+		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
+	}
+	if rl.captureErrorOutput {
+		rl.Addf("logging error output: %q\n", string(b))
+	}
+	return rl.w.Write(b)
+}
+
+// WriteHeader implements http.ResponseWriter.
+func (rl *respLogger) WriteHeader(status int) {
+	rl.recordStatus(status)
+	rl.w.WriteHeader(status)
+}
+
+func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	rl.hijacked = true
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Hijacker if the inner object (rl.w) implements http.Hijacker.
+	return rl.w.(http.Hijacker).Hijack()
+}
+
+func (rl *respLogger) recordStatus(status int) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.status = status
+	rl.statusRecorded = true
+	if rl.logStacktracePred(status) {
+		// Only log stacks for errors
+		stack := make([]byte, 50*1024)
+		stack = stack[:runtime.Stack(stack, false)]
+		rl.statusStack = "\n" + string(stack)
+		rl.captureErrorOutput = true
+	} else {
+		rl.statusStack = ""
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/lifecycle_signals.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"sync"
+)
+
+/*
+We make an attempt here to identify the events that take place during
+lifecycle of the apiserver.
+
+We also identify each event with a name so we can refer to it.
+
+Events:
+- ShutdownInitiated: KILL signal received
+- AfterShutdownDelayDuration: shutdown delay duration has passed
+- InFlightRequestsDrained: all in flight request(s) have been drained
+- HasBeenReady is signaled when the readyz endpoint succeeds for the first time
+
+The following is a sequence of shutdown events that we expect to see with
+  'ShutdownSendRetryAfter' = false:
+
+T0: ShutdownInitiated: KILL signal received
+	- /readyz starts returning red
+    - run pre shutdown hooks
+
+T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
+	- the default value of 'ShutdownDelayDuration' is '70s'
+	- it's time to initiate shutdown of the HTTP Server, server.Shutdown is invoked
+	- as a consequene, the Close function has is called for all listeners
+ 	- the HTTP Server stops listening immediately
+	- any new request arriving on a new TCP socket is denied with
+      a network error similar to 'connection refused'
+    - the HTTP Server waits gracefully for existing requests to complete
+      up to '60s' (dictated by ShutdownTimeout)
+	- active long running requests will receive a GOAWAY.
+
+T0+70s: HTTPServerStoppedListening:
+	- this event is signaled when the HTTP Server has stopped listening
+      which is immediately after server.Shutdown has been invoked
+
+T0 + 70s + up-to 60s: InFlightRequestsDrained: existing in flight requests have been drained
+	- long running requests are outside of this scope
+	- up-to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
+      any request in flight has a hard timeout of 60s.
+	- it's time to call 'Shutdown' on the audit events since all
+	  in flight request(s) have drained.
+
+
+The following is a sequence of shutdown events that we expect to see with
+  'ShutdownSendRetryAfter' = true:
+
+T0: ShutdownInitiated: KILL signal received
+	- /readyz starts returning red
+    - run pre shutdown hooks
+
+T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
+	- the default value of 'ShutdownDelayDuration' is '70s'
+	- the HTTP Server will continue to listen
+	- the apiserver is not accepting new request(s)
+		- it includes new request(s) on a new or an existing TCP connection
+		- new request(s) arriving after this point are replied with a 429
+      	  and the  response headers: 'Retry-After: 1` and 'Connection: close'
+	- note: these new request(s) will not show up in audit logs
+
+T0 + 70s + up to 60s: InFlightRequestsDrained: existing in flight requests have been drained
+	- long running requests are outside of this scope
+	- up to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
+      any request in flight has a hard timeout of 60s.
+	- server.Shutdown is called, the HTTP Server stops listening immediately
+    - the HTTP Server waits gracefully for existing requests to complete
+      up to '2s' (it's hard coded right now)
+*/
+
+// lifecycleSignal encapsulates a named apiserver event
+type lifecycleSignal interface {
+	// Signal signals the event, indicating that the event has occurred.
+	// Signal is idempotent, once signaled the event stays signaled and
+	// it immediately unblocks any goroutine waiting for this event.
+	Signal()
+
+	// Signaled returns a channel that is closed when the underlying event
+	// has been signaled. Successive calls to Signaled return the same value.
+	Signaled() <-chan struct{}
+
+	// Name returns the name of the signal, useful for logging.
+	Name() string
+}
+
+// lifecycleSignals provides an abstraction of the events that
+// transpire during the lifecycle of the apiserver. This abstraction makes it easy
+// for us to write unit tests that can verify expected graceful termination behavior.
+//
+// GenericAPIServer can use these to either:
+//   - signal that a particular termination event has transpired
+//   - wait for a designated termination event to transpire and do some action.
+type lifecycleSignals struct {
+	// ShutdownInitiated event is signaled when an apiserver shutdown has been initiated.
+	// It is signaled when the `stopCh` provided by the main goroutine
+	// receives a KILL signal and is closed as a consequence.
+	ShutdownInitiated lifecycleSignal
+
+	// AfterShutdownDelayDuration event is signaled as soon as ShutdownDelayDuration
+	// has elapsed since the ShutdownInitiated event.
+	// ShutdownDelayDuration allows the apiserver to delay shutdown for some time.
+	AfterShutdownDelayDuration lifecycleSignal
+
+	// PreShutdownHooksStopped event is signaled when all registered
+	// preshutdown hook(s) have finished running.
+	PreShutdownHooksStopped lifecycleSignal
+
+	// NotAcceptingNewRequest event is signaled when the server is no
+	// longer accepting any new request, from this point on any new
+	// request will receive an error.
+	NotAcceptingNewRequest lifecycleSignal
+
+	// InFlightRequestsDrained event is signaled when the existing requests
+	// in flight have completed. This is used as signal to shut down the audit backends
+	InFlightRequestsDrained lifecycleSignal
+
+	// HTTPServerStoppedListening termination event is signaled when the
+	// HTTP Server has stopped listening to the underlying socket.
+	HTTPServerStoppedListening lifecycleSignal
+
+	// HasBeenReady is signaled when the readyz endpoint succeeds for the first time.
+	HasBeenReady lifecycleSignal
+
+	// MuxAndDiscoveryComplete is signaled when all known HTTP paths have been installed.
+	// It exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler.
+	// The actual logic is implemented by an APIServer using the generic server library.
+	MuxAndDiscoveryComplete lifecycleSignal
+}
+
+// ShuttingDown returns the lifecycle signal that is signaled when
+// the server is not accepting any new requests.
+// this is the lifecycle event that is exported to the request handler
+// logic to indicate that the server is shutting down.
+func (s lifecycleSignals) ShuttingDown() <-chan struct{} {
+	return s.NotAcceptingNewRequest.Signaled()
+}
+
+// newLifecycleSignals returns an instance of lifecycleSignals interface to be used
+// to coordinate lifecycle of the apiserver
+func newLifecycleSignals() lifecycleSignals {
+	return lifecycleSignals{
+		ShutdownInitiated:          newNamedChannelWrapper("ShutdownInitiated"),
+		AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"),
+		PreShutdownHooksStopped:    newNamedChannelWrapper("PreShutdownHooksStopped"),
+		NotAcceptingNewRequest:     newNamedChannelWrapper("NotAcceptingNewRequest"),
+		InFlightRequestsDrained:    newNamedChannelWrapper("InFlightRequestsDrained"),
+		HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"),
+		HasBeenReady:               newNamedChannelWrapper("HasBeenReady"),
+		MuxAndDiscoveryComplete:    newNamedChannelWrapper("MuxAndDiscoveryComplete"),
+	}
+}
+
+func newNamedChannelWrapper(name string) lifecycleSignal {
+	return &namedChannelWrapper{
+		name: name,
+		once: sync.Once{},
+		ch:   make(chan struct{}),
+	}
+}
+
+type namedChannelWrapper struct {
+	name string
+	once sync.Once
+	ch   chan struct{}
+}
+
+func (e *namedChannelWrapper) Signal() {
+	e.once.Do(func() {
+		close(e.ch)
+	})
+}
+
+func (e *namedChannelWrapper) Signaled() <-chan struct{} {
+	return e.ch
+}
+
+func (e *namedChannelWrapper) Name() string {
+	return e.name
+}

e2e/vendor/k8s.io/apiserver/pkg/server/mux/OWNERS

@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts

e2e/vendor/k8s.io/apiserver/pkg/server/mux/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mux contains abstractions for http multiplexing of APIs.
+package mux // import "k8s.io/apiserver/pkg/server/mux"

e2e/vendor/k8s.io/apiserver/pkg/server/mux/pathrecorder.go

@@ -0,0 +1,281 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mux
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// PathRecorderMux wraps a mux object and records the registered exposedPaths.
+type PathRecorderMux struct {
+	// name is used for logging so you can trace requests through
+	name string
+
+	lock            sync.Mutex
+	notFoundHandler http.Handler
+	pathToHandler   map[string]http.Handler
+	prefixToHandler map[string]http.Handler
+
+	// mux stores a pathHandler and is used to handle the actual serving.
+	// Turns out, we want to accept trailing slashes, BUT we don't care about handling
+	// everything under them.  This does exactly matches only unless its explicitly requested to
+	// do something different
+	mux atomic.Value
+
+	// exposedPaths is the list of paths that should be shown at /
+	exposedPaths []string
+
+	// pathStacks holds the stacks of all registered paths.  This allows us to show a more helpful message
+	// before the "http: multiple registrations for %s" panic.
+	pathStacks map[string]string
+}
+
+// pathHandler is an http.Handler that will satisfy requests first by exact match, then by prefix,
+// then by notFoundHandler
+type pathHandler struct {
+	// muxName is used for logging so you can trace requests through
+	muxName string
+
+	// pathToHandler is a map of exactly matching request to its handler
+	pathToHandler map[string]http.Handler
+
+	// this has to be sorted by most slashes then by length
+	prefixHandlers []prefixHandler
+
+	// notFoundHandler is the handler to use for satisfying requests with no other match
+	notFoundHandler http.Handler
+}
+
+// prefixHandler holds the prefix it should match and the handler to use
+type prefixHandler struct {
+	// prefix is the prefix to test for a request match
+	prefix string
+	// handler is used to satisfy matching requests
+	handler http.Handler
+}
+
+// NewPathRecorderMux creates a new PathRecorderMux
+func NewPathRecorderMux(name string) *PathRecorderMux {
+	ret := &PathRecorderMux{
+		name:            name,
+		pathToHandler:   map[string]http.Handler{},
+		prefixToHandler: map[string]http.Handler{},
+		mux:             atomic.Value{},
+		exposedPaths:    []string{},
+		pathStacks:      map[string]string{},
+	}
+
+	ret.mux.Store(&pathHandler{notFoundHandler: http.NotFoundHandler()})
+	return ret
+}
+
+// ListedPaths returns the registered handler exposedPaths.
+func (m *PathRecorderMux) ListedPaths() []string {
+	m.lock.Lock()
+	handledPaths := append([]string{}, m.exposedPaths...)
+	m.lock.Unlock()
+
+	sort.Strings(handledPaths)
+	return handledPaths
+}
+
+func (m *PathRecorderMux) trackCallers(path string) {
+	stack := string(debug.Stack())
+	if existingStack, ok := m.pathStacks[path]; ok {
+		utilruntime.HandleError(fmt.Errorf("duplicate path registration of %q: original registration from %v\n\nnew registration from %v", path, existingStack, stack))
+	}
+	m.pathStacks[path] = stack
+}
+
+// refreshMuxLocked creates a new mux and must be called while locked.  Otherwise the view of handlers may
+// not be consistent
+func (m *PathRecorderMux) refreshMuxLocked() {
+	newMux := &pathHandler{
+		muxName:         m.name,
+		pathToHandler:   map[string]http.Handler{},
+		prefixHandlers:  []prefixHandler{},
+		notFoundHandler: http.NotFoundHandler(),
+	}
+	if m.notFoundHandler != nil {
+		newMux.notFoundHandler = m.notFoundHandler
+	}
+	for path, handler := range m.pathToHandler {
+		newMux.pathToHandler[path] = handler
+	}
+
+	keys := sets.StringKeySet(m.prefixToHandler).List()
+	sort.Sort(sort.Reverse(byPrefixPriority(keys)))
+	for _, prefix := range keys {
+		newMux.prefixHandlers = append(newMux.prefixHandlers, prefixHandler{
+			prefix:  prefix,
+			handler: m.prefixToHandler[prefix],
+		})
+	}
+
+	m.mux.Store(newMux)
+}
+
+// NotFoundHandler sets the handler to use if there's no match for a give path
+func (m *PathRecorderMux) NotFoundHandler(notFoundHandler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	m.notFoundHandler = notFoundHandler
+
+	m.refreshMuxLocked()
+}
+
+// Unregister removes a path from the mux.
+func (m *PathRecorderMux) Unregister(path string) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	delete(m.pathToHandler, path)
+	delete(m.prefixToHandler, path)
+	delete(m.pathStacks, path)
+	for i := range m.exposedPaths {
+		if m.exposedPaths[i] == path {
+			m.exposedPaths = append(m.exposedPaths[:i], m.exposedPaths[i+1:]...)
+			break
+		}
+	}
+
+	m.refreshMuxLocked()
+}
+
+// Handle registers the handler for the given pattern.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) Handle(path string, handler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.exposedPaths = append(m.exposedPaths, path)
+	m.pathToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// HandleFunc registers the handler function for the given pattern.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) HandleFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	m.Handle(path, http.HandlerFunc(handler))
+}
+
+// UnlistedHandle registers the handler for the given pattern, but doesn't list it.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) UnlistedHandle(path string, handler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.pathToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// UnlistedHandleFunc registers the handler function for the given pattern, but doesn't list it.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) UnlistedHandleFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	m.UnlistedHandle(path, http.HandlerFunc(handler))
+}
+
+// HandlePrefix is like Handle, but matches for anything under the path.  Like a standard golang trailing slash.
+func (m *PathRecorderMux) HandlePrefix(path string, handler http.Handler) {
+	if !strings.HasSuffix(path, "/") {
+		panic(fmt.Sprintf("%q must end in a trailing slash", path))
+	}
+
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.exposedPaths = append(m.exposedPaths, path)
+	m.prefixToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// UnlistedHandlePrefix is like UnlistedHandle, but matches for anything under the path.  Like a standard golang trailing slash.
+func (m *PathRecorderMux) UnlistedHandlePrefix(path string, handler http.Handler) {
+	if !strings.HasSuffix(path, "/") {
+		panic(fmt.Sprintf("%q must end in a trailing slash", path))
+	}
+
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.prefixToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// ServeHTTP makes it an http.Handler
+func (m *PathRecorderMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	m.mux.Load().(*pathHandler).ServeHTTP(w, r)
+}
+
+// ServeHTTP makes it an http.Handler
+func (h *pathHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if exactHandler, ok := h.pathToHandler[r.URL.Path]; ok {
+		klog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path)
+		exactHandler.ServeHTTP(w, r)
+		return
+	}
+
+	for _, prefixHandler := range h.prefixHandlers {
+		if strings.HasPrefix(r.URL.Path, prefixHandler.prefix) {
+			klog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix)
+			prefixHandler.handler.ServeHTTP(w, r)
+			return
+		}
+	}
+
+	klog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path)
+	h.notFoundHandler.ServeHTTP(w, r)
+}
+
+// byPrefixPriority sorts url prefixes by the order in which they should be tested by the mux
+// this has to be sorted by most slashes then by length so that we can iterate straight
+// through to match the "best" one first.
+type byPrefixPriority []string
+
+func (s byPrefixPriority) Len() int      { return len(s) }
+func (s byPrefixPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byPrefixPriority) Less(i, j int) bool {
+	lhsNumParts := strings.Count(s[i], "/")
+	rhsNumParts := strings.Count(s[j], "/")
+	if lhsNumParts != rhsNumParts {
+		return lhsNumParts < rhsNumParts
+	}
+
+	lhsLen := len(s[i])
+	rhsLen := len(s[j])
+	if lhsLen != rhsLen {
+		return lhsLen < rhsLen
+	}
+
+	return strings.Compare(s[i], s[j]) < 0
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/OWNERS

@@ -0,0 +1,15 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - enj
+reviewers:
+  - smarterclayton
+  - wojtek-t
+  - deads2k
+  - liggitt
+  - sttts
+  - soltysh
+  - dims
+  - cjcullen
+  - ping035627
+  - enj

e2e/vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -0,0 +1,255 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilwait "k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/admission/initializer"
+	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
+	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
+	mutatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/mutating"
+	validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
+	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
+	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
+	apiserverapiv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
+	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	"k8s.io/apiserver/pkg/server"
+	cacheddiscovery "k8s.io/client-go/discovery/cached/memory"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/restmapper"
+	"k8s.io/component-base/featuregate"
+)
+
+var configScheme = runtime.NewScheme()
+
+func init() {
+	utilruntime.Must(apiserverapi.AddToScheme(configScheme))
+	utilruntime.Must(apiserverapiv1alpha1.AddToScheme(configScheme))
+	utilruntime.Must(apiserverapiv1.AddToScheme(configScheme))
+}
+
+// AdmissionOptions holds the admission options
+type AdmissionOptions struct {
+	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
+	RecommendedPluginOrder []string
+	// DefaultOffPlugins is a set of plugin names that is disabled by default
+	DefaultOffPlugins sets.Set[string]
+
+	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
+	EnablePlugins []string
+	// DisablePlugins indicates plugins to be disabled passed through `--disable-admission-plugins`.
+	DisablePlugins []string
+	// ConfigFile is the file path with admission control configuration.
+	ConfigFile string
+	// Plugins contains all registered plugins.
+	Plugins *admission.Plugins
+	// Decorators is a list of admission decorator to wrap around the admission plugins
+	Decorators admission.Decorators
+}
+
+// NewAdmissionOptions creates a new instance of AdmissionOptions
+// Note:
+//
+//	In addition it calls RegisterAllAdmissionPlugins to register
+//	all generic admission plugins.
+//
+//	Provides the list of RecommendedPluginOrder that holds sane values
+//	that can be used by servers that don't care about admission chain.
+//	Servers that do care can overwrite/append that field after creation.
+func NewAdmissionOptions() *AdmissionOptions {
+	options := &AdmissionOptions{
+		Plugins:    admission.NewPlugins(),
+		Decorators: admission.Decorators{admission.DecoratorFunc(admissionmetrics.WithControllerMetrics)},
+		// This list is mix of mutating admission plugins and validating
+		// admission plugins. The apiserver always runs the validating ones
+		// after all the mutating ones, so their relative order in this list
+		// doesn't matter.
+		RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingadmissionpolicy.PluginName, mutatingwebhook.PluginName, validatingadmissionpolicy.PluginName, validatingwebhook.PluginName},
+		DefaultOffPlugins:      sets.Set[string]{},
+	}
+	server.RegisterAllAdmissionPlugins(options.Plugins)
+	return options
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
+	if a == nil {
+		return
+	}
+
+	fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+
+		"admission plugins that should be enabled in addition to default enabled ones ("+
+		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
+		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
+		"The order of plugins in this flag does not matter.")
+	fs.StringSliceVar(&a.DisablePlugins, "disable-admission-plugins", a.DisablePlugins, ""+
+		"admission plugins that should be disabled although they are in the default enabled plugins list ("+
+		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
+		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
+		"The order of plugins in this flag does not matter.")
+	fs.StringVar(&a.ConfigFile, "admission-control-config-file", a.ConfigFile,
+		"File with admission control configuration.")
+}
+
+// ApplyTo adds the admission chain to the server configuration.
+// In case admission plugin names were not provided by a cluster-admin they will be prepared from the recommended/default values.
+// In addition the method lazily initializes a generic plugin that is appended to the list of pluginInitializers
+// note this method uses:
+//
+//	genericconfig.Authorizer
+func (a *AdmissionOptions) ApplyTo(
+	c *server.Config,
+	informers informers.SharedInformerFactory,
+	kubeClient kubernetes.Interface,
+	dynamicClient dynamic.Interface,
+	features featuregate.FeatureGate,
+	pluginInitializers ...admission.PluginInitializer,
+) error {
+	if a == nil {
+		return nil
+	}
+
+	// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
+	if informers == nil {
+		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
+	}
+	if kubeClient == nil || dynamicClient == nil {
+		return fmt.Errorf("admission depends on a Kubernetes core API client, it cannot be nil")
+	}
+
+	pluginNames := a.enabledPluginNames()
+
+	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
+	if err != nil {
+		return fmt.Errorf("failed to read plugin config: %v", err)
+	}
+
+	discoveryClient := cacheddiscovery.NewMemCacheClient(kubeClient.Discovery())
+	discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
+	genericInitializer := initializer.New(kubeClient, dynamicClient, informers, c.Authorization.Authorizer, features,
+		c.DrainedNotify(), discoveryRESTMapper)
+	initializersChain := admission.PluginInitializers{genericInitializer}
+	initializersChain = append(initializersChain, pluginInitializers...)
+
+	admissionPostStartHook := func(hookContext server.PostStartHookContext) error {
+		discoveryRESTMapper.Reset()
+		go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, hookContext.Done())
+		return nil
+	}
+
+	err = c.AddPostStartHook("start-apiserver-admission-initializer", admissionPostStartHook)
+	if err != nil {
+		return fmt.Errorf("failed to add post start hook for policy admission: %w", err)
+	}
+
+	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
+	if err != nil {
+		return err
+	}
+
+	c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
+	return nil
+}
+
+// Validate verifies flags passed to AdmissionOptions.
+func (a *AdmissionOptions) Validate() []error {
+	if a == nil {
+		return nil
+	}
+
+	errs := []error{}
+
+	registeredPlugins := sets.NewString(a.Plugins.Registered()...)
+	for _, name := range a.EnablePlugins {
+		if !registeredPlugins.Has(name) {
+			errs = append(errs, fmt.Errorf("enable-admission-plugins plugin %q is unknown", name))
+		}
+	}
+
+	for _, name := range a.DisablePlugins {
+		if !registeredPlugins.Has(name) {
+			errs = append(errs, fmt.Errorf("disable-admission-plugins plugin %q is unknown", name))
+		}
+	}
+
+	enablePlugins := sets.NewString(a.EnablePlugins...)
+	disablePlugins := sets.NewString(a.DisablePlugins...)
+	if len(enablePlugins.Intersection(disablePlugins).List()) > 0 {
+		errs = append(errs, fmt.Errorf("%v in enable-admission-plugins and disable-admission-plugins "+
+			"overlapped", enablePlugins.Intersection(disablePlugins).List()))
+	}
+
+	// Verify RecommendedPluginOrder.
+	recommendPlugins := sets.NewString(a.RecommendedPluginOrder...)
+	intersections := registeredPlugins.Intersection(recommendPlugins)
+	if !intersections.Equal(recommendPlugins) {
+		// Developer error, this should never run in.
+		errs = append(errs, fmt.Errorf("plugins %v in RecommendedPluginOrder are not registered",
+			recommendPlugins.Difference(intersections).List()))
+	}
+	if !intersections.Equal(registeredPlugins) {
+		// Developer error, this should never run in.
+		errs = append(errs, fmt.Errorf("plugins %v registered are not in RecommendedPluginOrder",
+			registeredPlugins.Difference(intersections).List()))
+	}
+
+	return errs
+}
+
+// enabledPluginNames makes use of RecommendedPluginOrder, DefaultOffPlugins,
+// EnablePlugins, DisablePlugins fields
+// to prepare a list of ordered plugin names that are enabled.
+func (a *AdmissionOptions) enabledPluginNames() []string {
+	allOffPlugins := append(sets.List[string](a.DefaultOffPlugins), a.DisablePlugins...)
+	disabledPlugins := sets.NewString(allOffPlugins...)
+	enabledPlugins := sets.NewString(a.EnablePlugins...)
+	disabledPlugins = disabledPlugins.Difference(enabledPlugins)
+
+	orderedPlugins := []string{}
+	for _, plugin := range a.RecommendedPluginOrder {
+		if !disabledPlugins.Has(plugin) {
+			orderedPlugins = append(orderedPlugins, plugin)
+		}
+	}
+
+	return orderedPlugins
+}
+
+// Return names of plugins which are enabled by default
+func (a *AdmissionOptions) defaultEnabledPluginNames() []string {
+	defaultOnPluginNames := []string{}
+	for _, pluginName := range a.RecommendedPluginOrder {
+		if !a.DefaultOffPlugins.Has(pluginName) {
+			defaultOnPluginNames = append(defaultOnPluginNames, pluginName)
+		}
+	}
+
+	return defaultOnPluginNames
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/resourceconfig"
+	serverstore "k8s.io/apiserver/pkg/server/storage"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+// APIEnablementOptions contains the options for which resources to turn on and off.
+// Given small aggregated API servers, this option isn't required for "normal" API servers
+type APIEnablementOptions struct {
+	RuntimeConfig cliflag.ConfigurationMap
+}
+
+func NewAPIEnablementOptions() *APIEnablementOptions {
+	return &APIEnablementOptions{
+		RuntimeConfig: make(cliflag.ConfigurationMap),
+	}
+}
+
+// AddFlags adds flags for a specific APIServer to the specified FlagSet
+func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+	fs.Var(&s.RuntimeConfig, "runtime-config", ""+
+		"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
+		"v1=true|false for the core API group\n"+
+		"<group>/<version>=true|false for a specific API group and version (e.g. apps/v1=true)\n"+
+		"api/all=true|false controls all API versions\n"+
+		"api/ga=true|false controls all API versions of the form v[0-9]+\n"+
+		"api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+\n"+
+		"api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+\n"+
+		"api/legacy is deprecated, and will be removed in a future version")
+}
+
+// Validate validates RuntimeConfig with a list of registries.
+// Usually this list only has one element, the apiserver registry of the process.
+// But in the advanced (and usually not recommended) case of delegated apiservers there can be more.
+// Validate will filter out the known groups of each registry.
+// If anything is left over after that, an error is returned.
+func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
+	if s == nil {
+		return nil
+	}
+
+	errors := []error{}
+	if s.RuntimeConfig[resourceconfig.APIAll] == "false" && len(s.RuntimeConfig) == 1 {
+		// Do not allow only set api/all=false, in such case apiserver startup has no meaning.
+		return append(errors, fmt.Errorf("invalid key with only %v=false", resourceconfig.APIAll))
+	}
+
+	groups, err := resourceconfig.ParseGroups(s.RuntimeConfig)
+	if err != nil {
+		return append(errors, err)
+	}
+
+	for _, registry := range registries {
+		// filter out known groups
+		groups = unknownGroups(groups, registry)
+	}
+	if len(groups) != 0 {
+		errors = append(errors, fmt.Errorf("unknown api groups %s", strings.Join(groups, ",")))
+	}
+
+	return errors
+}
+
+// ApplyTo override MergedResourceConfig with defaults and registry
+func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
+	if s == nil {
+		return nil
+	}
+
+	mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(defaultResourceConfig, s.RuntimeConfig, registry)
+	c.MergedResourceConfig = mergedResourceConfig
+
+	return err
+}
+
+func unknownGroups(groups []string, registry GroupRegistry) []string {
+	unknownGroups := []string{}
+	for _, group := range groups {
+		if !registry.IsGroupRegistered(group) {
+			unknownGroups = append(unknownGroups, group)
+		}
+	}
+	return unknownGroups
+}
+
+// GroupRegistry provides a method to check whether given group is registered.
+type GroupRegistry interface {
+	// IsRegistered returns true if given group is registered.
+	IsGroupRegistered(group string) bool
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -0,0 +1,612 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"k8s.io/klog/v2"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/sets"
+	auditinternal "k8s.io/apiserver/pkg/apis/audit"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/audit/policy"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/webhook"
+	pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
+	pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
+	plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
+	pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
+)
+
+const (
+	// Default configuration values for ModeBatch.
+	defaultBatchBufferSize = 10000 // Buffer up to 10000 events before starting discarding.
+	// These batch parameters are only used by the webhook backend.
+	defaultBatchMaxSize       = 400              // Only send up to 400 events at a time.
+	defaultBatchMaxWait       = 30 * time.Second // Send events at least twice a minute.
+	defaultBatchThrottleQPS   = 10               // Limit the send rate by 10 QPS.
+	defaultBatchThrottleBurst = 15               // Allow up to 15 QPS burst.
+)
+
+func appendBackend(existing, newBackend audit.Backend) audit.Backend {
+	if existing == nil {
+		return newBackend
+	}
+	if newBackend == nil {
+		return existing
+	}
+	return audit.Union(existing, newBackend)
+}
+
+type AuditOptions struct {
+	// Policy configuration file for filtering audit events that are captured.
+	// If unspecified, a default is provided.
+	PolicyFile string
+
+	// Plugin options
+	LogOptions     AuditLogOptions
+	WebhookOptions AuditWebhookOptions
+}
+
+const (
+	// ModeBatch indicates that the audit backend should buffer audit events
+	// internally, sending batch updates either once a certain number of
+	// events have been received or a certain amount of time has passed.
+	ModeBatch = "batch"
+	// ModeBlocking causes the audit backend to block on every attempt to process
+	// a set of events. This causes requests to the API server to wait for the
+	// flush before sending a response.
+	ModeBlocking = "blocking"
+	// ModeBlockingStrict is the same as ModeBlocking, except when there is
+	// a failure during audit logging at RequestReceived stage, the whole
+	// request to apiserver will fail.
+	ModeBlockingStrict = "blocking-strict"
+)
+
+// AllowedModes is the modes known for audit backends.
+var AllowedModes = []string{
+	ModeBatch,
+	ModeBlocking,
+	ModeBlockingStrict,
+}
+
+type AuditBatchOptions struct {
+	// Should the backend asynchronous batch events to the webhook backend or
+	// should the backend block responses?
+	//
+	// Defaults to asynchronous batch events.
+	Mode string
+	// Configuration for batching backend. Only used in batch mode.
+	BatchConfig pluginbuffered.BatchConfig
+}
+
+type AuditTruncateOptions struct {
+	// Whether truncating is enabled or not.
+	Enabled bool
+
+	// Truncating configuration.
+	TruncateConfig plugintruncate.Config
+}
+
+// AuditLogOptions determines the output of the structured audit log by default.
+type AuditLogOptions struct {
+	Path       string
+	MaxAge     int
+	MaxBackups int
+	MaxSize    int
+	Format     string
+	Compress   bool
+
+	BatchOptions    AuditBatchOptions
+	TruncateOptions AuditTruncateOptions
+
+	// API group version used for serializing audit events.
+	GroupVersionString string
+}
+
+// AuditWebhookOptions control the webhook configuration for audit events.
+type AuditWebhookOptions struct {
+	ConfigFile     string
+	InitialBackoff time.Duration
+
+	BatchOptions    AuditBatchOptions
+	TruncateOptions AuditTruncateOptions
+
+	// API group version used for serializing audit events.
+	GroupVersionString string
+}
+
+func NewAuditOptions() *AuditOptions {
+	return &AuditOptions{
+		WebhookOptions: AuditWebhookOptions{
+			InitialBackoff: pluginwebhook.DefaultInitialBackoffDelay,
+			BatchOptions: AuditBatchOptions{
+				Mode:        ModeBatch,
+				BatchConfig: defaultWebhookBatchConfig(),
+			},
+			TruncateOptions:    NewAuditTruncateOptions(),
+			GroupVersionString: "audit.k8s.io/v1",
+		},
+		LogOptions: AuditLogOptions{
+			Format: pluginlog.FormatJson,
+			BatchOptions: AuditBatchOptions{
+				Mode:        ModeBlocking,
+				BatchConfig: defaultLogBatchConfig(),
+			},
+			TruncateOptions:    NewAuditTruncateOptions(),
+			GroupVersionString: "audit.k8s.io/v1",
+		},
+	}
+}
+
+func NewAuditTruncateOptions() AuditTruncateOptions {
+	return AuditTruncateOptions{
+		Enabled: false,
+		TruncateConfig: plugintruncate.Config{
+			MaxBatchSize: 10 * 1024 * 1024, // 10MB
+			MaxEventSize: 100 * 1024,       // 100KB
+		},
+	}
+}
+
+// Validate checks invalid config combination
+func (o *AuditOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	var allErrors []error
+	allErrors = append(allErrors, o.LogOptions.Validate()...)
+	allErrors = append(allErrors, o.WebhookOptions.Validate()...)
+
+	return allErrors
+}
+
+func validateBackendMode(pluginName string, mode string) error {
+	for _, m := range AllowedModes {
+		if m == mode {
+			return nil
+		}
+	}
+	return fmt.Errorf("invalid audit %s mode %s, allowed modes are %q", pluginName, mode, strings.Join(AllowedModes, ","))
+}
+
+func validateBackendBatchOptions(pluginName string, options AuditBatchOptions) error {
+	if err := validateBackendMode(pluginName, options.Mode); err != nil {
+		return err
+	}
+	if options.Mode != ModeBatch {
+		// Don't validate the unused options.
+		return nil
+	}
+	config := options.BatchConfig
+	if config.BufferSize <= 0 {
+		return fmt.Errorf("invalid audit batch %s buffer size %v, must be a positive number", pluginName, config.BufferSize)
+	}
+	if config.MaxBatchSize <= 0 {
+		return fmt.Errorf("invalid audit batch %s max batch size %v, must be a positive number", pluginName, config.MaxBatchSize)
+	}
+	if config.ThrottleEnable {
+		if config.ThrottleQPS <= 0 {
+			return fmt.Errorf("invalid audit batch %s throttle QPS %v, must be a positive number", pluginName, config.ThrottleQPS)
+		}
+		if config.ThrottleBurst <= 0 {
+			return fmt.Errorf("invalid audit batch %s throttle burst %v, must be a positive number", pluginName, config.ThrottleBurst)
+		}
+	}
+	return nil
+}
+
+var knownGroupVersions = []schema.GroupVersion{
+	auditv1.SchemeGroupVersion,
+}
+
+func validateGroupVersionString(groupVersion string) error {
+	gv, err := schema.ParseGroupVersion(groupVersion)
+	if err != nil {
+		return err
+	}
+	if !knownGroupVersion(gv) {
+		return fmt.Errorf("invalid group version, allowed versions are %q", knownGroupVersions)
+	}
+	if gv != auditv1.SchemeGroupVersion {
+		klog.Warningf("%q is deprecated and will be removed in a future release, use %q instead", gv, auditv1.SchemeGroupVersion)
+	}
+	return nil
+}
+
+func knownGroupVersion(gv schema.GroupVersion) bool {
+	for _, knownGv := range knownGroupVersions {
+		if gv == knownGv {
+			return true
+		}
+	}
+	return false
+}
+
+func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.PolicyFile, "audit-policy-file", o.PolicyFile,
+		"Path to the file that defines the audit policy configuration.")
+
+	o.LogOptions.AddFlags(fs)
+	o.LogOptions.BatchOptions.AddFlags(pluginlog.PluginName, fs)
+	o.LogOptions.TruncateOptions.AddFlags(pluginlog.PluginName, fs)
+	o.WebhookOptions.AddFlags(fs)
+	o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs)
+	o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs)
+}
+
+func (o *AuditOptions) ApplyTo(
+	c *server.Config,
+) error {
+	if o == nil {
+		return nil
+	}
+	if c == nil {
+		return fmt.Errorf("server config must be non-nil")
+	}
+
+	// 1. Build policy evaluator
+	evaluator, err := o.newPolicyRuleEvaluator()
+	if err != nil {
+		return err
+	}
+
+	// 2. Build log backend
+	var logBackend audit.Backend
+	w, err := o.LogOptions.getWriter()
+	if err != nil {
+		return err
+	}
+	if w != nil {
+		if evaluator == nil {
+			klog.V(2).Info("No audit policy file provided, no events will be recorded for log backend")
+		} else {
+			logBackend = o.LogOptions.newBackend(w)
+		}
+	}
+
+	// 3. Build webhook backend
+	var webhookBackend audit.Backend
+	if o.WebhookOptions.enabled() {
+		if evaluator == nil {
+			klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend")
+		} else {
+			if c.EgressSelector != nil {
+				var egressDialer utilnet.DialFunc
+				egressDialer, err = c.EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext())
+				if err != nil {
+					return err
+				}
+				webhookBackend, err = o.WebhookOptions.newUntruncatedBackend(egressDialer)
+			} else {
+				webhookBackend, err = o.WebhookOptions.newUntruncatedBackend(nil)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	groupVersion, err := schema.ParseGroupVersion(o.WebhookOptions.GroupVersionString)
+	if err != nil {
+		return err
+	}
+
+	// 4. Apply dynamic options.
+	var dynamicBackend audit.Backend
+	if webhookBackend != nil {
+		// if only webhook is enabled wrap it in the truncate options
+		dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
+	}
+
+	// 5. Set the policy rule evaluator
+	c.AuditPolicyRuleEvaluator = evaluator
+
+	// 6. Join the log backend with the webhooks
+	c.AuditBackend = appendBackend(logBackend, dynamicBackend)
+
+	if c.AuditBackend != nil {
+		klog.V(2).Infof("Using audit backend: %s", c.AuditBackend)
+	}
+	return nil
+}
+
+func (o *AuditOptions) newPolicyRuleEvaluator() (audit.PolicyRuleEvaluator, error) {
+	if o.PolicyFile == "" {
+		return nil, nil
+	}
+
+	p, err := policy.LoadPolicyFromFile(o.PolicyFile)
+	if err != nil {
+		return nil, fmt.Errorf("loading audit policy file: %v", err)
+	}
+	return policy.NewPolicyRuleEvaluator(p), nil
+}
+
+func (o *AuditBatchOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
+	fs.StringVar(&o.Mode, fmt.Sprintf("audit-%s-mode", pluginName), o.Mode,
+		"Strategy for sending audit events. Blocking indicates sending events should block"+
+			" server responses. Batch causes the backend to buffer and write events"+
+			" asynchronously. Known modes are "+strings.Join(AllowedModes, ",")+".")
+	fs.IntVar(&o.BatchConfig.BufferSize, fmt.Sprintf("audit-%s-batch-buffer-size", pluginName),
+		o.BatchConfig.BufferSize, "The size of the buffer to store events before "+
+			"batching and writing. Only used in batch mode.")
+	fs.IntVar(&o.BatchConfig.MaxBatchSize, fmt.Sprintf("audit-%s-batch-max-size", pluginName),
+		o.BatchConfig.MaxBatchSize, "The maximum size of a batch. Only used in batch mode.")
+	fs.DurationVar(&o.BatchConfig.MaxBatchWait, fmt.Sprintf("audit-%s-batch-max-wait", pluginName),
+		o.BatchConfig.MaxBatchWait, "The amount of time to wait before force writing the "+
+			"batch that hadn't reached the max size. Only used in batch mode.")
+	fs.BoolVar(&o.BatchConfig.ThrottleEnable, fmt.Sprintf("audit-%s-batch-throttle-enable", pluginName),
+		o.BatchConfig.ThrottleEnable, "Whether batching throttling is enabled. Only used in batch mode.")
+	fs.Float32Var(&o.BatchConfig.ThrottleQPS, fmt.Sprintf("audit-%s-batch-throttle-qps", pluginName),
+		o.BatchConfig.ThrottleQPS, "Maximum average number of batches per second. "+
+			"Only used in batch mode.")
+	fs.IntVar(&o.BatchConfig.ThrottleBurst, fmt.Sprintf("audit-%s-batch-throttle-burst", pluginName),
+		o.BatchConfig.ThrottleBurst, "Maximum number of requests sent at the same "+
+			"moment if ThrottleQPS was not utilized before. Only used in batch mode.")
+}
+
+type ignoreErrorsBackend struct {
+	audit.Backend
+}
+
+func (i *ignoreErrorsBackend) ProcessEvents(ev ...*auditinternal.Event) bool {
+	i.Backend.ProcessEvents(ev...)
+	return true
+}
+
+func (i *ignoreErrorsBackend) String() string {
+	return fmt.Sprintf("ignoreErrors<%s>", i.Backend)
+}
+
+func (o *AuditBatchOptions) wrapBackend(delegate audit.Backend) audit.Backend {
+	if o.Mode == ModeBlockingStrict {
+		return delegate
+	}
+	if o.Mode == ModeBlocking {
+		return &ignoreErrorsBackend{Backend: delegate}
+	}
+	return pluginbuffered.NewBackend(delegate, o.BatchConfig)
+}
+
+func (o *AuditTruncateOptions) Validate(pluginName string) error {
+	config := o.TruncateConfig
+	if config.MaxEventSize <= 0 {
+		return fmt.Errorf("invalid audit truncate %s max event size %v, must be a positive number", pluginName, config.MaxEventSize)
+	}
+	if config.MaxBatchSize < config.MaxEventSize {
+		return fmt.Errorf("invalid audit truncate %s max batch size %v, must be greater than "+
+			"max event size (%v)", pluginName, config.MaxBatchSize, config.MaxEventSize)
+	}
+	return nil
+}
+
+func (o *AuditTruncateOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
+	fs.BoolVar(&o.Enabled, fmt.Sprintf("audit-%s-truncate-enabled", pluginName),
+		o.Enabled, "Whether event and batch truncating is enabled.")
+	fs.Int64Var(&o.TruncateConfig.MaxBatchSize, fmt.Sprintf("audit-%s-truncate-max-batch-size", pluginName),
+		o.TruncateConfig.MaxBatchSize, "Maximum size of the batch sent to the underlying backend. "+
+			"Actual serialized size can be several hundreds of bytes greater. If a batch exceeds this limit, "+
+			"it is split into several batches of smaller size.")
+	fs.Int64Var(&o.TruncateConfig.MaxEventSize, fmt.Sprintf("audit-%s-truncate-max-event-size", pluginName),
+		o.TruncateConfig.MaxEventSize, "Maximum size of the audit event sent to the underlying backend. "+
+			"If the size of an event is greater than this number, first request and response are removed, and "+
+			"if this doesn't reduce the size enough, event is discarded.")
+}
+
+func (o *AuditTruncateOptions) wrapBackend(delegate audit.Backend, gv schema.GroupVersion) audit.Backend {
+	if !o.Enabled {
+		return delegate
+	}
+	return plugintruncate.NewBackend(delegate, o.TruncateConfig, gv)
+}
+
+func (o *AuditLogOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.Path, "audit-log-path", o.Path,
+		"If set, all requests coming to the apiserver will be logged to this file.  '-' means standard out.")
+	fs.IntVar(&o.MaxAge, "audit-log-maxage", o.MaxAge,
+		"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
+	fs.IntVar(&o.MaxBackups, "audit-log-maxbackup", o.MaxBackups,
+		"The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files.")
+	fs.IntVar(&o.MaxSize, "audit-log-maxsize", o.MaxSize,
+		"The maximum size in megabytes of the audit log file before it gets rotated.")
+	fs.StringVar(&o.Format, "audit-log-format", o.Format,
+		"Format of saved audits. \"legacy\" indicates 1-line text format for each event."+
+			" \"json\" indicates structured json format. Known formats are "+
+			strings.Join(pluginlog.AllowedFormats, ",")+".")
+	fs.StringVar(&o.GroupVersionString, "audit-log-version", o.GroupVersionString,
+		"API group and version used for serializing audit events written to log.")
+	fs.BoolVar(&o.Compress, "audit-log-compress", o.Compress, "If set, the rotated log files will be compressed using gzip.")
+}
+
+func (o *AuditLogOptions) Validate() []error {
+	// Check whether the log backend is enabled based on the options.
+	if !o.enabled() {
+		return nil
+	}
+
+	var allErrors []error
+
+	if err := validateBackendBatchOptions(pluginlog.PluginName, o.BatchOptions); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := o.TruncateOptions.Validate(pluginlog.PluginName); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	if err := validateGroupVersionString(o.GroupVersionString); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	// Check log format
+	if !sets.NewString(pluginlog.AllowedFormats...).Has(o.Format) {
+		allErrors = append(allErrors, fmt.Errorf("invalid audit log format %s, allowed formats are %q", o.Format, strings.Join(pluginlog.AllowedFormats, ",")))
+	}
+
+	// Check validities of MaxAge, MaxBackups and MaxSize of log options, if file log backend is enabled.
+	if o.MaxAge < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxage %v can't be a negative number", o.MaxAge))
+	}
+	if o.MaxBackups < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxbackup %v can't be a negative number", o.MaxBackups))
+	}
+	if o.MaxSize < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxsize %v can't be a negative number", o.MaxSize))
+	}
+
+	return allErrors
+}
+
+// Check whether the log backend is enabled based on the options.
+func (o *AuditLogOptions) enabled() bool {
+	return o != nil && o.Path != ""
+}
+
+func (o *AuditLogOptions) getWriter() (io.Writer, error) {
+	if !o.enabled() {
+		return nil, nil
+	}
+
+	if o.Path == "-" {
+		return os.Stdout, nil
+	}
+
+	if err := o.ensureLogFile(); err != nil {
+		return nil, fmt.Errorf("ensureLogFile: %w", err)
+	}
+
+	return &lumberjack.Logger{
+		Filename:   o.Path,
+		MaxAge:     o.MaxAge,
+		MaxBackups: o.MaxBackups,
+		MaxSize:    o.MaxSize,
+		Compress:   o.Compress,
+	}, nil
+}
+
+func (o *AuditLogOptions) ensureLogFile() error {
+	if err := os.MkdirAll(filepath.Dir(o.Path), 0700); err != nil {
+		return err
+	}
+	mode := os.FileMode(0600)
+	f, err := os.OpenFile(o.Path, os.O_CREATE|os.O_APPEND|os.O_RDWR, mode)
+	if err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+func (o *AuditLogOptions) newBackend(w io.Writer) audit.Backend {
+	groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
+	log := pluginlog.NewBackend(w, o.Format, groupVersion)
+	log = o.BatchOptions.wrapBackend(log)
+	log = o.TruncateOptions.wrapBackend(log, groupVersion)
+	return log
+}
+
+func (o *AuditWebhookOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.ConfigFile, "audit-webhook-config-file", o.ConfigFile,
+		"Path to a kubeconfig formatted file that defines the audit webhook configuration.")
+	fs.DurationVar(&o.InitialBackoff, "audit-webhook-initial-backoff",
+		o.InitialBackoff, "The amount of time to wait before retrying the first failed request.")
+	fs.DurationVar(&o.InitialBackoff, "audit-webhook-batch-initial-backoff",
+		o.InitialBackoff, "The amount of time to wait before retrying the first failed request.")
+	fs.MarkDeprecated("audit-webhook-batch-initial-backoff",
+		"Deprecated, use --audit-webhook-initial-backoff instead.")
+	fs.StringVar(&o.GroupVersionString, "audit-webhook-version", o.GroupVersionString,
+		"API group and version used for serializing audit events written to webhook.")
+}
+
+func (o *AuditWebhookOptions) Validate() []error {
+	if !o.enabled() {
+		return nil
+	}
+
+	var allErrors []error
+	if err := validateBackendBatchOptions(pluginwebhook.PluginName, o.BatchOptions); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := o.TruncateOptions.Validate(pluginwebhook.PluginName); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	if err := validateGroupVersionString(o.GroupVersionString); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	return allErrors
+}
+
+func (o *AuditWebhookOptions) enabled() bool {
+	return o != nil && o.ConfigFile != ""
+}
+
+// newUntruncatedBackend returns a webhook backend without the truncate options applied
+// this is done so that the same trucate backend can wrap both the webhook and dynamic backends
+func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc) (audit.Backend, error) {
+	groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
+	webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, webhook.DefaultRetryBackoffWithInitialDelay(o.InitialBackoff), customDial)
+	if err != nil {
+		return nil, fmt.Errorf("initializing audit webhook: %v", err)
+	}
+	webhook = o.BatchOptions.wrapBackend(webhook)
+	return webhook, nil
+}
+
+// defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend.
+func defaultWebhookBatchConfig() pluginbuffered.BatchConfig {
+	return pluginbuffered.BatchConfig{
+		BufferSize:   defaultBatchBufferSize,
+		MaxBatchSize: defaultBatchMaxSize,
+		MaxBatchWait: defaultBatchMaxWait,
+
+		ThrottleEnable: true,
+		ThrottleQPS:    defaultBatchThrottleQPS,
+		ThrottleBurst:  defaultBatchThrottleBurst,
+
+		AsyncDelegate: true,
+	}
+}
+
+// defaultLogBatchConfig returns the default BatchConfig used by the Log backend.
+func defaultLogBatchConfig() pluginbuffered.BatchConfig {
+	return pluginbuffered.BatchConfig{
+		BufferSize: defaultBatchBufferSize,
+		// Batching is not useful for the log-file backend.
+		// MaxBatchWait ignored.
+		MaxBatchSize:   1,
+		ThrottleEnable: false,
+		// Asynchronous log threads just create lock contention.
+		AsyncDelegate: false,
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -0,0 +1,497 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
+)
+
+// DefaultAuthWebhookRetryBackoff is the default backoff parameters for
+// both authentication and authorization webhook used by the apiserver.
+func DefaultAuthWebhookRetryBackoff() *wait.Backoff {
+	return &wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   1.5,
+		Jitter:   0.2,
+		Steps:    5,
+	}
+}
+
+type RequestHeaderAuthenticationOptions struct {
+	// ClientCAFile is the root certificate bundle to verify client certificates on incoming requests
+	// before trusting usernames in headers.
+	ClientCAFile string
+
+	UsernameHeaders     []string
+	UIDHeaders          []string
+	GroupHeaders        []string
+	ExtraHeaderPrefixes []string
+	AllowedNames        []string
+}
+
+func (s *RequestHeaderAuthenticationOptions) Validate() []error {
+	allErrors := []error{}
+
+	if err := checkForWhiteSpaceOnly("requestheader-username-headers", s.UsernameHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-group-headers", s.GroupHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-extra-headers-prefix", s.ExtraHeaderPrefixes...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-allowed-names", s.AllowedNames...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	if len(s.UsernameHeaders) > 0 && !caseInsensitiveHas(s.UsernameHeaders, "X-Remote-User") {
+		klog.Warningf("--requestheader-username-headers is set without specifying the standard X-Remote-User header - API aggregation will not work")
+	}
+	if len(s.GroupHeaders) > 0 && !caseInsensitiveHas(s.GroupHeaders, "X-Remote-Group") {
+		klog.Warningf("--requestheader-group-headers is set without specifying the standard X-Remote-Group header - API aggregation will not work")
+	}
+	if len(s.ExtraHeaderPrefixes) > 0 && !caseInsensitiveHas(s.ExtraHeaderPrefixes, "X-Remote-Extra-") {
+		klog.Warningf("--requestheader-extra-headers-prefix is set without specifying the standard X-Remote-Extra- header prefix - API aggregation will not work")
+	}
+
+	if !utilfeature.DefaultFeatureGate.Enabled(features.RemoteRequestHeaderUID) {
+		if len(s.UIDHeaders) > 0 {
+			allErrors = append(allErrors, fmt.Errorf("--requestheader-uid-headers requires the %q feature to be enabled", features.RemoteRequestHeaderUID))
+		}
+	} else {
+		if err := checkForWhiteSpaceOnly("requestheader-uid-headers", s.UIDHeaders...); err != nil {
+			allErrors = append(allErrors, err)
+		}
+		if len(s.UIDHeaders) > 0 && !caseInsensitiveHas(s.UIDHeaders, "X-Remote-Uid") {
+			// this was added later and so we are able to error out
+			allErrors = append(allErrors, fmt.Errorf("--requestheader-uid-headers is set without specifying the standard X-Remote-Uid header - API aggregation will not work"))
+		}
+	}
+
+	return allErrors
+}
+
+func checkForWhiteSpaceOnly(flag string, headerNames ...string) error {
+	for _, headerName := range headerNames {
+		if len(strings.TrimSpace(headerName)) == 0 {
+			return fmt.Errorf("empty value in %q", flag)
+		}
+	}
+
+	return nil
+}
+
+func caseInsensitiveHas(headers []string, header string) bool {
+	for _, h := range headers {
+		if strings.EqualFold(h, header) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.StringSliceVar(&s.UsernameHeaders, "requestheader-username-headers", s.UsernameHeaders, ""+
+		"List of request headers to inspect for usernames. X-Remote-User is common.")
+
+	fs.StringSliceVar(&s.UIDHeaders, "requestheader-uid-headers", s.UIDHeaders, ""+
+		"List of request headers to inspect for UIDs. X-Remote-Uid is suggested. Requires the RemoteRequestHeaderUID feature to be enabled.")
+
+	fs.StringSliceVar(&s.GroupHeaders, "requestheader-group-headers", s.GroupHeaders, ""+
+		"List of request headers to inspect for groups. X-Remote-Group is suggested.")
+
+	fs.StringSliceVar(&s.ExtraHeaderPrefixes, "requestheader-extra-headers-prefix", s.ExtraHeaderPrefixes, ""+
+		"List of request header prefixes to inspect. X-Remote-Extra- is suggested.")
+
+	fs.StringVar(&s.ClientCAFile, "requestheader-client-ca-file", s.ClientCAFile, ""+
+		"Root certificate bundle to use to verify client certificates on incoming requests "+
+		"before trusting usernames in headers specified by --requestheader-username-headers. "+
+		"WARNING: generally do not depend on authorization being already done for incoming requests.")
+
+	fs.StringSliceVar(&s.AllowedNames, "requestheader-allowed-names", s.AllowedNames, ""+
+		"List of client certificate common names to allow to provide usernames in headers "+
+		"specified by --requestheader-username-headers. If empty, any client certificate validated "+
+		"by the authorities in --requestheader-client-ca-file is allowed.")
+}
+
+// ToAuthenticationRequestHeaderConfig returns a RequestHeaderConfig config object for these options
+// if necessary, nil otherwise.
+func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() (*authenticatorfactory.RequestHeaderConfig, error) {
+	if len(s.ClientCAFile) == 0 {
+		return nil, nil
+	}
+
+	caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &authenticatorfactory.RequestHeaderConfig{
+		UsernameHeaders:     headerrequest.StaticStringSlice(s.UsernameHeaders),
+		UIDHeaders:          headerrequest.StaticStringSlice(s.UIDHeaders),
+		GroupHeaders:        headerrequest.StaticStringSlice(s.GroupHeaders),
+		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
+		CAContentProvider:   caBundleProvider,
+		AllowedClientNames:  headerrequest.StaticStringSlice(s.AllowedNames),
+	}, nil
+}
+
+// ClientCertAuthenticationOptions provides different options for client cert auth. You should use `GetClientVerifyOptionFn` to
+// get the verify options for your authenticator.
+type ClientCertAuthenticationOptions struct {
+	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
+	ClientCA string
+
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// Generally this is the CA bundle file used to authenticate client certificates
+	// If non-nil, this takes priority over the ClientCA file.
+	CAContentProvider dynamiccertificates.CAContentProvider
+}
+
+// GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
+func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
+	if s.CAContentProvider != nil {
+		return s.CAContentProvider, nil
+	}
+
+	if len(s.ClientCA) == 0 {
+		return nil, nil
+	}
+
+	return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
+}
+
+func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
+		"If set, any request presenting a client certificate signed by one of "+
+		"the authorities in the client-ca-file is authenticated with an identity "+
+		"corresponding to the CommonName of the client certificate.")
+}
+
+// DelegatingAuthenticationOptions provides an easy way for composing API servers to delegate their authentication to
+// the root kube API server.  The API federator will act as
+// a front proxy and direction connections will be able to delegate to the core kube API server
+type DelegatingAuthenticationOptions struct {
+	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
+	// TokenAccessReview.authentication.k8s.io endpoint for checking tokens.
+	RemoteKubeConfigFile string
+	// RemoteKubeConfigFileOptional is specifying whether not specifying the kubeconfig or
+	// a missing in-cluster config will be fatal.
+	RemoteKubeConfigFileOptional bool
+
+	// CacheTTL is the length of time that a token authentication answer will be cached.
+	CacheTTL time.Duration
+
+	ClientCert    ClientCertAuthenticationOptions
+	RequestHeader RequestHeaderAuthenticationOptions
+
+	// SkipInClusterLookup indicates missing authentication configuration should not be retrieved from the cluster configmap
+	SkipInClusterLookup bool
+
+	// TolerateInClusterLookupFailure indicates failures to look up authentication configuration from the cluster configmap should not be fatal.
+	// Setting this can result in an authenticator that will reject all requests.
+	TolerateInClusterLookupFailure bool
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
+	// TokenRequestTimeout specifies a time limit for requests made by the authorization webhook client.
+	// The default value is set to 10 seconds.
+	TokenRequestTimeout time.Duration
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
+
+	// Anonymous gives user an option to enable/disable Anonymous authentication.
+	Anonymous *apiserver.AnonymousAuthConfig
+}
+
+func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
+	return &DelegatingAuthenticationOptions{
+		// very low for responsiveness, but high enough to handle storms
+		CacheTTL:   10 * time.Second,
+		ClientCert: ClientCertAuthenticationOptions{},
+		RequestHeader: RequestHeaderAuthenticationOptions{
+			UsernameHeaders: []string{"x-remote-user"},
+			// we specifically don't default UID headers as these were introduced
+			// later (kube 1.32) and we don't want 3rd parties to be trusting the default headers
+			// before we can safely say that all KAS instances know they should
+			// remove them from an incoming request in its WithAuthentication handler.
+			// The defaulting will be enabled in a future (1.33+) version.
+			UIDHeaders:          nil,
+			GroupHeaders:        []string{"x-remote-group"},
+			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
+		},
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
+		TokenRequestTimeout: 10 * time.Second,
+		Anonymous:           &apiserver.AnonymousAuthConfig{Enabled: true},
+	}
+}
+
+// WithCustomRetryBackoff sets the custom backoff parameters for the authentication webhook retry logic.
+func (s *DelegatingAuthenticationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
+// WithRequestTimeout sets the given timeout for requests made by the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithRequestTimeout(timeout time.Duration) {
+	s.TokenRequestTimeout = timeout
+}
+
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
+func (s *DelegatingAuthenticationOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	allErrors = append(allErrors, s.RequestHeader.Validate()...)
+
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
+	return allErrors
+}
+
+func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	var optionalKubeConfigSentence string
+	if s.RemoteKubeConfigFileOptional {
+		optionalKubeConfigSentence = " This is optional. If empty, all token requests are considered to be anonymous and no client CA is looked up in the cluster."
+	}
+	fs.StringVar(&s.RemoteKubeConfigFile, "authentication-kubeconfig", s.RemoteKubeConfigFile, ""+
+		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+
+		"tokenreviews.authentication.k8s.io."+optionalKubeConfigSentence)
+
+	fs.DurationVar(&s.CacheTTL, "authentication-token-webhook-cache-ttl", s.CacheTTL,
+		"The duration to cache responses from the webhook token authenticator.")
+
+	s.ClientCert.AddFlags(fs)
+	s.RequestHeader.AddFlags(fs)
+
+	fs.BoolVar(&s.SkipInClusterLookup, "authentication-skip-lookup", s.SkipInClusterLookup, ""+
+		"If false, the authentication-kubeconfig will be used to lookup missing authentication "+
+		"configuration from the cluster.")
+	fs.BoolVar(&s.TolerateInClusterLookupFailure, "authentication-tolerate-lookup-failure", s.TolerateInClusterLookupFailure, ""+
+		"If true, failures to look up missing authentication configuration from the cluster are not considered fatal. "+
+		"Note that this can result in authentication that treats all requests as anonymous.")
+}
+
+func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
+	if s == nil {
+		authenticationInfo.Authenticator = nil
+		return nil
+	}
+
+	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
+		Anonymous:                &apiserver.AnonymousAuthConfig{Enabled: true},
+		CacheTTL:                 s.CacheTTL,
+		WebhookRetryBackoff:      s.WebhookRetryBackoff,
+		TokenAccessReviewTimeout: s.TokenRequestTimeout,
+	}
+
+	client, err := s.getClient()
+	if err != nil {
+		return fmt.Errorf("failed to get delegated authentication kubeconfig: %v", err)
+	}
+
+	// configure token review
+	if client != nil {
+		cfg.TokenAccessReviewClient = client.AuthenticationV1()
+	}
+
+	// get the clientCA information
+	clientCASpecified := s.ClientCert != ClientCertAuthenticationOptions{}
+	var clientCAProvider dynamiccertificates.CAContentProvider
+	if clientCASpecified {
+		clientCAProvider, err = s.ClientCert.GetClientCAContentProvider()
+		if err != nil {
+			return fmt.Errorf("unable to load client CA provider: %v", err)
+		}
+		cfg.ClientCertificateCAContentProvider = clientCAProvider
+		if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to assign client CA provider: %v", err)
+		}
+
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			clientCAProvider, err = dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "client-ca-file", client)
+			if err != nil {
+				return fmt.Errorf("unable to load configmap based client CA file: %v", err)
+			}
+			cfg.ClientCertificateCAContentProvider = clientCAProvider
+			if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+				return fmt.Errorf("unable to assign configmap based client CA file: %v", err)
+			}
+
+		}
+	}
+
+	requestHeaderCAFileSpecified := len(s.RequestHeader.ClientCAFile) > 0
+	var requestHeaderConfig *authenticatorfactory.RequestHeaderConfig
+	if requestHeaderCAFileSpecified {
+		requestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		if err != nil {
+			return fmt.Errorf("unable to create request header authentication config: %v", err)
+		}
+
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			requestHeaderConfig, err = s.createRequestHeaderConfig(client)
+			if err != nil {
+				if s.TolerateInClusterLookupFailure {
+					klog.Warningf("Error looking up in-cluster authentication configuration: %v", err)
+					klog.Warning("Continuing without authentication configuration. This may treat all requests as anonymous.")
+					klog.Warning("To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false")
+				} else {
+					return fmt.Errorf("unable to load configmap based request-header-client-ca-file: %v", err)
+				}
+			}
+		}
+	}
+	if requestHeaderConfig != nil {
+		cfg.RequestHeaderConfig = requestHeaderConfig
+		authenticationInfo.RequestHeaderConfig = requestHeaderConfig
+		if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to load request-header-client-ca-file: %v", err)
+		}
+	}
+
+	// create authenticator
+	authenticator, securityDefinitions, err := cfg.New()
+	if err != nil {
+		return err
+	}
+	authenticationInfo.Authenticator = authenticator
+	if openAPIConfig != nil {
+		openAPIConfig.SecurityDefinitions = securityDefinitions
+	}
+
+	return nil
+}
+
+const (
+	authenticationConfigMapNamespace = metav1.NamespaceSystem
+	// authenticationConfigMapName is the name of ConfigMap in the kube-system namespace holding the root certificate
+	// bundle to use to verify client certificates on incoming requests before trusting usernames in headers specified
+	// by --requestheader-username-headers. This is created in the cluster by the kube-apiserver.
+	// "WARNING: generally do not depend on authorization being already done for incoming requests.")
+	authenticationConfigMapName = "extension-apiserver-authentication"
+)
+
+func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
+	dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
+	}
+
+	//  look up authentication configuration in the cluster and in case of an err defer to authentication-tolerate-lookup-failure flag
+	//  We are passing the context to ProxyCerts.RunOnce as it needs to implement RunOnce(ctx) however the
+	//  context is not used at all. So passing a empty context shouldn't be a problem
+	ctx := context.TODO()
+	if err := dynamicRequestHeaderProvider.RunOnce(ctx); err != nil {
+		return nil, err
+	}
+
+	return &authenticatorfactory.RequestHeaderConfig{
+		CAContentProvider:   dynamicRequestHeaderProvider,
+		UsernameHeaders:     headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)),
+		UIDHeaders:          headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UIDHeaders)),
+		GroupHeaders:        headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)),
+		ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)),
+		AllowedClientNames:  headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)),
+	}, nil
+}
+
+// getClient returns a Kubernetes clientset. If s.RemoteKubeConfigFileOptional is true, nil will be returned
+// if no kubeconfig is specified by the user and the in-cluster config is not found.
+func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, error) {
+	var clientConfig *rest.Config
+	var err error
+	if len(s.RemoteKubeConfigFile) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		clientConfig, err = loader.ClientConfig()
+	} else {
+		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
+		// use this path. If it is optional, ignore errors.
+		clientConfig, err = rest.InClusterConfig()
+		if err != nil && s.RemoteKubeConfigFileOptional {
+			if err != rest.ErrNotInCluster {
+				klog.Warningf("failed to read in-cluster kubeconfig for delegated authentication: %v", err)
+			}
+			return nil, nil
+		}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get delegated authentication kubeconfig: %v", err)
+	}
+
+	// set high qps/burst limits since this will effectively limit API server responsiveness
+	clientConfig.QPS = 200
+	clientConfig.Burst = 400
+	// do not set a timeout on the http client, instead use context for cancellation
+	// if multiple timeouts were set, the request will pick the smaller timeout to be applied, leaving other useless.
+	//
+	// see https://github.com/golang/go/blob/a937729c2c2f6950a32bc5cd0f5b88700882f078/src/net/http/client.go#L364
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
+
+	return kubernetes.NewForConfig(clientConfig)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/authentication_dynamic_request_header.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ dynamiccertificates.ControllerRunner = &DynamicRequestHeaderController{}
+var _ dynamiccertificates.CAContentProvider = &DynamicRequestHeaderController{}
+
+var _ headerrequest.RequestHeaderAuthRequestProvider = &DynamicRequestHeaderController{}
+
+// DynamicRequestHeaderController combines DynamicCAFromConfigMapController and RequestHeaderAuthRequestController
+// into one controller for dynamically filling RequestHeaderConfig struct
+type DynamicRequestHeaderController struct {
+	*dynamiccertificates.ConfigMapCAController
+	*headerrequest.RequestHeaderAuthRequestController
+}
+
+// newDynamicRequestHeaderController creates a new controller that implements DynamicRequestHeaderController
+func newDynamicRequestHeaderController(client kubernetes.Interface) (*DynamicRequestHeaderController, error) {
+	requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+		"client-ca",
+		authenticationConfigMapNamespace,
+		authenticationConfigMapName,
+		"requestheader-client-ca-file",
+		client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err)
+	}
+
+	requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController(
+		authenticationConfigMapName,
+		authenticationConfigMapNamespace,
+		client,
+		"requestheader-username-headers",
+		"requestheader-uid-headers",
+		"requestheader-group-headers",
+		"requestheader-extra-headers-prefix",
+		"requestheader-allowed-names",
+	)
+	return &DynamicRequestHeaderController{
+		ConfigMapCAController:              requestHeaderCAController,
+		RequestHeaderAuthRequestController: requestHeaderAuthRequestController,
+	}, nil
+}
+
+func (c *DynamicRequestHeaderController) RunOnce(ctx context.Context) error {
+	errs := []error{}
+	errs = append(errs, c.ConfigMapCAController.RunOnce(ctx))
+	errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce(ctx))
+	return errors.NewAggregate(errs)
+}
+
+func (c *DynamicRequestHeaderController) Run(ctx context.Context, workers int) {
+	go c.ConfigMapCAController.Run(ctx, workers)
+	go c.RequestHeaderAuthRequestController.Run(ctx, workers)
+	<-ctx.Done()
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -0,0 +1,243 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/authorization/path"
+	"k8s.io/apiserver/pkg/authorization/union"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+)
+
+// DelegatingAuthorizationOptions provides an easy way for composing API servers to delegate their authorization to
+// the root kube API server.
+// WARNING: never assume that every authenticated incoming request already does authorization.
+// The aggregator in the kube API server does this today, but this behaviour is not
+// guaranteed in the future.
+type DelegatingAuthorizationOptions struct {
+	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
+	// SubjectAccessReview.authorization.k8s.io endpoint for checking tokens.
+	RemoteKubeConfigFile string
+	// RemoteKubeConfigFileOptional is specifying whether not specifying the kubeconfig or
+	// a missing in-cluster config will be fatal.
+	RemoteKubeConfigFileOptional bool
+
+	// AllowCacheTTL is the length of time that a successful authorization response will be cached
+	AllowCacheTTL time.Duration
+
+	// DenyCacheTTL is the length of time that an unsuccessful authorization response will be cached.
+	// You generally want more responsive, "deny, try again" flows.
+	DenyCacheTTL time.Duration
+
+	// AlwaysAllowPaths are HTTP paths which are excluded from authorization. They can be plain
+	// paths or end in * in which case prefix-match is applied. A leading / is optional.
+	AlwaysAllowPaths []string
+
+	// AlwaysAllowGroups are groups which are allowed to take any actions.  In kube, this is system:masters.
+	AlwaysAllowGroups []string
+
+	// ClientTimeout specifies a time limit for requests made by SubjectAccessReviews client.
+	// The default value is set to 10 seconds.
+	ClientTimeout time.Duration
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
+}
+
+func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions {
+	return &DelegatingAuthorizationOptions{
+		// very low for responsiveness, but high enough to handle storms
+		AllowCacheTTL:       10 * time.Second,
+		DenyCacheTTL:        10 * time.Second,
+		ClientTimeout:       10 * time.Second,
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
+		// This allows the kubelet to always get health and readiness without causing an authorization check.
+		// This field can be cleared by callers if they don't want this behavior.
+		AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"},
+		// In an authorization call delegated to a kube-apiserver (the expected common-case), system:masters has full
+		// authority in a hard-coded authorizer.  This means that our default can reasonably be to skip an authorization
+		// check for system:masters.
+		// This field can be cleared by callers if they don't want this behavior.
+		AlwaysAllowGroups: []string{"system:masters"},
+	}
+}
+
+// WithAlwaysAllowGroups appends the list of paths to AlwaysAllowGroups
+func (s *DelegatingAuthorizationOptions) WithAlwaysAllowGroups(groups ...string) *DelegatingAuthorizationOptions {
+	s.AlwaysAllowGroups = append(s.AlwaysAllowGroups, groups...)
+	return s
+}
+
+// WithAlwaysAllowPaths appends the list of paths to AlwaysAllowPaths
+func (s *DelegatingAuthorizationOptions) WithAlwaysAllowPaths(paths ...string) *DelegatingAuthorizationOptions {
+	s.AlwaysAllowPaths = append(s.AlwaysAllowPaths, paths...)
+	return s
+}
+
+// WithClientTimeout sets the given timeout for SAR client used by this authorizer
+func (s *DelegatingAuthorizationOptions) WithClientTimeout(timeout time.Duration) {
+	s.ClientTimeout = timeout
+}
+
+// WithCustomRetryBackoff sets the custom backoff parameters for the authorization webhook retry logic.
+func (s *DelegatingAuthorizationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+func (s *DelegatingAuthorizationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
+func (s *DelegatingAuthorizationOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
+	return allErrors
+}
+
+func (s *DelegatingAuthorizationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	var optionalKubeConfigSentence string
+	if s.RemoteKubeConfigFileOptional {
+		optionalKubeConfigSentence = " This is optional. If empty, all requests not skipped by authorization are forbidden."
+	}
+	fs.StringVar(&s.RemoteKubeConfigFile, "authorization-kubeconfig", s.RemoteKubeConfigFile,
+		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+
+			"subjectaccessreviews.authorization.k8s.io."+optionalKubeConfigSentence)
+
+	fs.DurationVar(&s.AllowCacheTTL, "authorization-webhook-cache-authorized-ttl",
+		s.AllowCacheTTL,
+		"The duration to cache 'authorized' responses from the webhook authorizer.")
+
+	fs.DurationVar(&s.DenyCacheTTL,
+		"authorization-webhook-cache-unauthorized-ttl", s.DenyCacheTTL,
+		"The duration to cache 'unauthorized' responses from the webhook authorizer.")
+
+	fs.StringSliceVar(&s.AlwaysAllowPaths, "authorization-always-allow-paths", s.AlwaysAllowPaths,
+		"A list of HTTP paths to skip during authorization, i.e. these are authorized without "+
+			"contacting the 'core' kubernetes server.")
+}
+
+func (s *DelegatingAuthorizationOptions) ApplyTo(c *server.AuthorizationInfo) error {
+	if s == nil {
+		c.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+		return nil
+	}
+
+	client, err := s.getClient()
+	if err != nil {
+		return err
+	}
+
+	c.Authorizer, err = s.toAuthorizer(client)
+	return err
+}
+
+func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interface) (authorizer.Authorizer, error) {
+	var authorizers []authorizer.Authorizer
+
+	if len(s.AlwaysAllowGroups) > 0 {
+		authorizers = append(authorizers, authorizerfactory.NewPrivilegedGroups(s.AlwaysAllowGroups...))
+	}
+
+	if len(s.AlwaysAllowPaths) > 0 {
+		a, err := path.NewAuthorizer(s.AlwaysAllowPaths)
+		if err != nil {
+			return nil, err
+		}
+		authorizers = append(authorizers, a)
+	}
+
+	if client == nil {
+		klog.Warning("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.")
+	} else {
+		cfg := authorizerfactory.DelegatingAuthorizerConfig{
+			SubjectAccessReviewClient: client.AuthorizationV1(),
+			AllowCacheTTL:             s.AllowCacheTTL,
+			DenyCacheTTL:              s.DenyCacheTTL,
+			WebhookRetryBackoff:       s.WebhookRetryBackoff,
+		}
+		delegatedAuthorizer, err := cfg.New()
+		if err != nil {
+			return nil, err
+		}
+		authorizers = append(authorizers, delegatedAuthorizer)
+	}
+
+	return union.New(authorizers...), nil
+}
+
+func (s *DelegatingAuthorizationOptions) getClient() (kubernetes.Interface, error) {
+	var clientConfig *rest.Config
+	var err error
+	if len(s.RemoteKubeConfigFile) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		clientConfig, err = loader.ClientConfig()
+	} else {
+		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
+		// use this path. If it is optional, ignore errors.
+		clientConfig, err = rest.InClusterConfig()
+		if err != nil && s.RemoteKubeConfigFileOptional {
+			if err != rest.ErrNotInCluster {
+				klog.Warningf("failed to read in-cluster kubeconfig for delegated authorization: %v", err)
+			}
+			return nil, nil
+		}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get delegated authorization kubeconfig: %v", err)
+	}
+
+	// set high qps/burst limits since this will effectively limit API server responsiveness
+	clientConfig.QPS = 200
+	clientConfig.Burst = 400
+	clientConfig.Timeout = s.ClientTimeout
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
+
+	return kubernetes.NewForConfig(clientConfig)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/coreapi.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/util/feature"
+	clientgoinformers "k8s.io/client-go/informers"
+	clientgoclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	tracing "k8s.io/component-base/tracing"
+)
+
+// CoreAPIOptions contains options to configure the connection to a core API Kubernetes apiserver.
+type CoreAPIOptions struct {
+	// CoreAPIKubeconfigPath is a filename for a kubeconfig file to contact the core API server with.
+	// If it is not set, the in cluster config is used.
+	CoreAPIKubeconfigPath string
+}
+
+func NewCoreAPIOptions() *CoreAPIOptions {
+	return &CoreAPIOptions{}
+}
+
+func (o *CoreAPIOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.CoreAPIKubeconfigPath, "kubeconfig", o.CoreAPIKubeconfigPath,
+		"kubeconfig file pointing at the 'core' kubernetes server.")
+}
+
+func (o *CoreAPIOptions) ApplyTo(config *server.RecommendedConfig) error {
+	if o == nil {
+		return nil
+	}
+
+	// create shared informer for Kubernetes APIs
+	var kubeconfig *rest.Config
+	var err error
+	if len(o.CoreAPIKubeconfigPath) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfigPath}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+		kubeconfig, err = loader.ClientConfig()
+		if err != nil {
+			return fmt.Errorf("failed to load kubeconfig at %q: %v", o.CoreAPIKubeconfigPath, err)
+		}
+	} else {
+		kubeconfig, err = rest.InClusterConfig()
+		if err != nil {
+			return err
+		}
+	}
+	if feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		kubeconfig.Wrap(tracing.WrapperFor(config.TracerProvider))
+	}
+	clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeconfig)
+	if err != nil {
+		return fmt.Errorf("failed to create Kubernetes clientset: %v", err)
+	}
+	config.ClientConfig = kubeconfig
+	config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
+
+	return nil
+}
+
+func (o *CoreAPIOptions) Validate() []error {
+	return nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/doc.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// package options is the public flags and options used by a generic api
+// server. It takes a minimal set of dependencies and does not reference
+// implementations, in order to ensure it may be reused by multiple components
+// (such as CLI commands that wish to generate or validate config).
+package options // import "k8s.io/apiserver/pkg/server/options"

e2e/vendor/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+	"k8s.io/utils/path"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+)
+
+// EgressSelectorOptions holds the api server egress selector options.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md
+type EgressSelectorOptions struct {
+	// ConfigFile is the file path with api-server egress selector configuration.
+	ConfigFile string
+}
+
+// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions
+//
+// The option is to point to a configuration file for egress/konnectivity.
+// This determines which types of requests use egress/konnectivity and how they use it.
+// If empty the API Server will attempt to connect directly using the network.
+func NewEgressSelectorOptions() *EgressSelectorOptions {
+	return &EgressSelectorOptions{}
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile,
+		"File with apiserver egress selector configuration.")
+}
+
+// ApplyTo adds the egress selector settings to the server configuration.
+// In case egress selector settings were not provided by a cluster-admin
+// they will be prepared from the recommended/default/no-op values.
+func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read egress selector config: %v", err)
+	}
+	errs := egressselector.ValidateEgressSelectorConfiguration(npConfig)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate())
+	}
+
+	cs, err := egressselector.NewEgressSelector(npConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err)
+	}
+	c.EgressSelector = cs
+	return nil
+}
+
+// Validate verifies flags passed to EgressSelectorOptions.
+func (o *EgressSelectorOptions) Validate() []error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+
+	errs := []error{}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists || err != nil {
+		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
+	}
+
+	return errs
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - sig-auth-encryption-at-rest-approvers
+reviewers:
+  - sig-auth-encryption-at-rest-reviewers
+labels:
+  - sig/auth

e2e/vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -0,0 +1,288 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// workqueueKey is the dummy key used to process change in encryption config file.
+const workqueueKey = "key"
+
+// EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed.
+var EncryptionConfigFileChangePollDuration = time.Minute
+
+// DynamicEncryptionConfigContent which can dynamically handle changes in encryption config file.
+type DynamicEncryptionConfigContent struct {
+	name string
+
+	// filePath is the path of the file to read.
+	filePath string
+
+	// lastLoadedEncryptionConfigHash stores last successfully read encryption config file content.
+	lastLoadedEncryptionConfigHash string
+
+	// queue for processing changes in encryption config file.
+	queue workqueue.TypedRateLimitingInterface[string]
+
+	// dynamicTransformers updates the transformers when encryption config file changes.
+	dynamicTransformers *encryptionconfig.DynamicTransformers
+
+	// identity of the api server
+	apiServerID string
+
+	// can be swapped during testing
+	getEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
+	loadEncryptionConfig    func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
+}
+
+func init() {
+	metrics.RegisterMetrics()
+}
+
+// NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
+func NewDynamicEncryptionConfiguration(
+	name, filePath string,
+	dynamicTransformers *encryptionconfig.DynamicTransformers,
+	configContentHash string,
+	apiServerID string,
+) *DynamicEncryptionConfigContent {
+	return &DynamicEncryptionConfigContent{
+		name:                           name,
+		filePath:                       filePath,
+		lastLoadedEncryptionConfigHash: configContentHash,
+		dynamicTransformers:            dynamicTransformers,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: name},
+		),
+		apiServerID: apiServerID,
+		getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
+			return encryptionconfig.GetEncryptionConfigHash(filepath)
+		},
+		loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig,
+	}
+}
+
+// Run starts the controller and blocks until ctx is canceled.
+func (d *DynamicEncryptionConfigContent) Run(ctx context.Context) {
+	defer utilruntime.HandleCrash()
+
+	klog.InfoS("Starting controller", "name", d.name)
+	defer klog.InfoS("Shutting down controller", "name", d.name)
+
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		defer d.queue.ShutDown()
+		<-ctx.Done()
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		d.runWorker(ctx)
+	}()
+
+	// this function polls changes in the encryption config file by placing a dummy key in the queue.
+	// the 'runWorker' function then picks up this dummy key and processes the changes.
+	// the goroutine terminates when 'ctx' is canceled.
+	_ = wait.PollUntilContextCancel(
+		ctx,
+		EncryptionConfigFileChangePollDuration,
+		true,
+		func(ctx context.Context) (bool, error) {
+			// add dummy item to the queue to trigger file content processing.
+			d.queue.Add(workqueueKey)
+
+			// return false to continue polling.
+			return false, nil
+		},
+	)
+
+	wg.Wait()
+}
+
+// runWorker to process file content
+func (d *DynamicEncryptionConfigContent) runWorker(ctx context.Context) {
+	for d.processNextWorkItem(ctx) {
+	}
+}
+
+// processNextWorkItem processes file content when there is a message in the queue.
+func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
+	// key here is dummy item in the queue to trigger file content processing.
+	key, quit := d.queue.Get()
+	if quit {
+		return false
+	}
+	defer d.queue.Done(key)
+
+	d.processWorkItem(serverCtx, key)
+
+	return true
+}
+
+func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey string) {
+	var (
+		updatedEffectiveConfig  bool
+		err                     error
+		encryptionConfiguration *encryptionconfig.EncryptionConfiguration
+		configChanged           bool
+	)
+
+	// get context to close the new transformers (on error cases and on the next reload)
+	// serverCtx is attached to the API server's lifecycle so we will always close transformers on shut down
+	ctx, closeTransformers := context.WithCancel(serverCtx)
+
+	defer func() {
+		// TODO can work queue metrics help here?
+
+		if !updatedEffectiveConfig {
+			// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
+			closeTransformers()
+		}
+
+		if updatedEffectiveConfig && err == nil {
+			metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
+		}
+
+		if err != nil {
+			metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
+			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
+			// add dummy item back to the queue to trigger file content processing.
+			d.queue.AddRateLimited(workqueueKey)
+		}
+	}()
+
+	encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
+	if err != nil {
+		return
+	}
+	if !configChanged {
+		return
+	}
+
+	if len(encryptionConfiguration.HealthChecks) != 1 {
+		err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
+		return
+	}
+	// get healthz checks for all new KMS plugins.
+	if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
+		return
+	}
+
+	// update transformers.
+	// when reload=true there must always be one healthz check.
+	d.dynamicTransformers.Set(
+		encryptionConfiguration.Transformers,
+		closeTransformers,
+		encryptionConfiguration.HealthChecks[0],
+		encryptionConfiguration.KMSCloseGracePeriod,
+	)
+
+	// update local copy of recent config content once update is successful.
+	d.lastLoadedEncryptionConfigHash = encryptionConfiguration.EncryptionFileContentHash
+	klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
+
+	updatedEffectiveConfig = true
+}
+
+// loadEncryptionConfig processes the next set of content from the file.
+func (d *DynamicEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
+	_ *encryptionconfig.EncryptionConfiguration,
+	configChanged bool,
+	_ error,
+) {
+	contentHash, err := d.getEncryptionConfigHash(ctx, d.filePath)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// check if encryptionConfig is different from the current. Do nothing if they are the same.
+	if contentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (before load)", "name", d.name)
+		return nil, false, nil
+	}
+
+	// this code path will only execute if reload=true. So passing true explicitly.
+	encryptionConfiguration, err := d.loadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// check if encryptionConfig is different from the current (again to avoid TOCTOU). Do nothing if they are the same.
+	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (after load)", "name", d.name)
+		return nil, false, nil
+	}
+
+	return encryptionConfiguration, true, nil
+}
+
+// minKMSPluginCloseGracePeriod can be lowered in unit tests to make the health check poll faster
+var minKMSPluginCloseGracePeriod = 10 * time.Second
+
+func (d *DynamicEncryptionConfigContent) validateNewTransformersHealth(
+	ctx context.Context,
+	kmsPluginHealthzCheck healthz.HealthChecker,
+	kmsPluginCloseGracePeriod time.Duration,
+) error {
+	// test if new transformers are healthy
+	var healthCheckError error
+
+	if kmsPluginCloseGracePeriod < minKMSPluginCloseGracePeriod {
+		kmsPluginCloseGracePeriod = minKMSPluginCloseGracePeriod
+	}
+
+	// really make sure that the immediate check does not hang
+	var cancel context.CancelFunc
+	ctx, cancel = context.WithTimeout(ctx, kmsPluginCloseGracePeriod)
+	defer cancel()
+
+	pollErr := wait.PollImmediateWithContext(ctx, 100*time.Millisecond, kmsPluginCloseGracePeriod, func(ctx context.Context) (bool, error) {
+		// create a fake http get request to health check endpoint
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("/healthz/%s", kmsPluginHealthzCheck.Name()), nil)
+		if err != nil {
+			return false, err
+		}
+
+		healthCheckError = kmsPluginHealthzCheck.Check(req)
+		return healthCheckError == nil, nil
+	})
+	if pollErr != nil {
+		return fmt.Errorf("health check for new transformers failed, polling error %v: %w", pollErr, healthCheckError)
+	}
+	klog.V(2).InfoS("Health check succeeded")
+	return nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -0,0 +1,131 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"hash"
+	"sync"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "encryption_config_controller"
+)
+
+var (
+	encryptionConfigAutomaticReloadsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reloads_total",
+			Help:           "Total number of reload successes and failures of encryption configuration split by apiserver identity.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status", "apiserver_id_hash"},
+	)
+
+	// deprecatedEncryptionConfigAutomaticReloadFailureTotal has been deprecated in 1.30.0
+	// use encryptionConfigAutomaticReloadsTotal instead
+	deprecatedEncryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "automatic_reload_failures_total",
+			Help:              "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.30.0",
+		},
+		[]string{"apiserver_id_hash"},
+	)
+
+	// deprecatedEncryptionConfigAutomaticReloadSuccessTotal has been deprecated in 1.30.0
+	// use encryptionConfigAutomaticReloadsTotal instead
+	deprecatedEncryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "automatic_reload_success_total",
+			Help:              "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.30.0",
+		},
+		[]string{"apiserver_id_hash"},
+	)
+
+	encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_last_timestamp_seconds",
+			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status", "apiserver_id_hash"},
+	)
+)
+
+var registerMetrics sync.Once
+var hashPool *sync.Pool
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		hashPool = &sync.Pool{
+			New: func() interface{} {
+				return sha256.New()
+			},
+		}
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadsTotal)
+		legacyregistry.MustRegister(deprecatedEncryptionConfigAutomaticReloadFailureTotal)
+		legacyregistry.MustRegister(deprecatedEncryptionConfigAutomaticReloadSuccessTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
+	})
+}
+
+func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadsTotal.WithLabelValues("failure", apiServerIDHash).Inc()
+	deprecatedEncryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
+}
+
+func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadsTotal.WithLabelValues("success", apiServerIDHash).Inc()
+	deprecatedEncryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
+}
+
+func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
+	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
+}
+
+func getHash(data string) string {
+	if len(data) == 0 {
+		return ""
+	}
+	h := hashPool.Get().(hash.Hash)
+	h.Reset()
+	h.Write([]byte(data))
+	dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
+	hashPool.Put(h)
+	return dataHash
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -0,0 +1,520 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
+	storagevalue "k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/klog/v2"
+)
+
+type EtcdOptions struct {
+	StorageConfig                           storagebackend.Config
+	EncryptionProviderConfigFilepath        string
+	EncryptionProviderConfigAutomaticReload bool
+
+	EtcdServersOverrides []string
+
+	// To enable protobuf as storage format, it is enough
+	// to set it to "application/vnd.kubernetes.protobuf".
+	DefaultStorageMediaType string
+	DeleteCollectionWorkers int
+	EnableGarbageCollection bool
+
+	// Set EnableWatchCache to false to disable all watch caches
+	EnableWatchCache bool
+	// Set DefaultWatchCacheSize to zero to disable watch caches for those resources that have no explicit cache size set
+	DefaultWatchCacheSize int
+	// WatchCacheSizes represents override to a given resource
+	WatchCacheSizes []string
+
+	// SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
+	// This allows multiple invocations of the Apply methods without duplication of said endpoints.
+	SkipHealthEndpoints bool
+}
+
+var storageTypes = sets.NewString(
+	storagebackend.StorageTypeETCD3,
+)
+
+func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
+	options := &EtcdOptions{
+		StorageConfig:           *backendConfig,
+		DefaultStorageMediaType: "application/json",
+		DeleteCollectionWorkers: 1,
+		EnableGarbageCollection: true,
+		EnableWatchCache:        true,
+		DefaultWatchCacheSize:   100,
+	}
+	options.StorageConfig.CountMetricPollPeriod = time.Minute
+	return options
+}
+
+var storageMediaTypes = sets.New(
+	runtime.ContentTypeJSON,
+	runtime.ContentTypeYAML,
+	runtime.ContentTypeProtobuf,
+)
+
+func (s *EtcdOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	if len(s.StorageConfig.Transport.ServerList) == 0 {
+		allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified"))
+	}
+
+	if s.StorageConfig.Type != storagebackend.StorageTypeUnset && !storageTypes.Has(s.StorageConfig.Type) {
+		allErrors = append(allErrors, fmt.Errorf("--storage-backend invalid, allowed values: %s. If not specified, it will default to 'etcd3'", strings.Join(storageTypes.List(), ", ")))
+	}
+
+	for _, override := range s.EtcdServersOverrides {
+		tokens := strings.Split(override, "#")
+		if len(tokens) != 2 {
+			allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
+			continue
+		}
+
+		apiresource := strings.Split(tokens[0], "/")
+		if len(apiresource) != 2 {
+			allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
+			continue
+		}
+
+	}
+
+	if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
+		allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
+	}
+
+	if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) {
+		allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", ")))
+	}
+
+	return allErrors
+}
+
+// AddFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet
+func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+
+		"Per-resource etcd servers overrides, comma separated. The individual override "+
+		"format: group/resource#servers, where servers are URLs, semicolon separated. "+
+		"Note that this applies only to resources compiled into this server binary. ")
+
+	fs.StringVar(&s.DefaultStorageMediaType, "storage-media-type", s.DefaultStorageMediaType, ""+
+		"The media type to use to store objects in storage. "+
+		"Some resources or storage backends may only support a specific media type and will ignore this setting. "+
+		"Supported media types: [application/json, application/yaml, application/vnd.kubernetes.protobuf]")
+	fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers,
+		"Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.")
+
+	fs.BoolVar(&s.EnableGarbageCollection, "enable-garbage-collector", s.EnableGarbageCollection, ""+
+		"Enables the generic garbage collector. MUST be synced with the corresponding flag "+
+		"of the kube-controller-manager.")
+
+	fs.BoolVar(&s.EnableWatchCache, "watch-cache", s.EnableWatchCache,
+		"Enable watch caching in the apiserver")
+
+	fs.IntVar(&s.DefaultWatchCacheSize, "default-watch-cache-size", s.DefaultWatchCacheSize,
+		"Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.")
+
+	fs.MarkDeprecated("default-watch-cache-size",
+		"watch caches are sized automatically and this flag will be removed in a future version")
+
+	fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+
+		"Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+
+		"The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+
+		"group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+
+		"and size is a number. This option is only meaningful for resources built into the apiserver, "+
+		"not ones defined by CRDs or aggregated from external servers, and is only consulted if the "+
+		"watch-cache is enabled. The only meaningful size setting to supply here is zero, which means to "+
+		"disable watch caching for the associated resource; all non-zero values are equivalent and mean "+
+		"to not disable watch caching for that resource")
+
+	fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type,
+		"The storage backend for persistence. Options: 'etcd3' (default).")
+
+	fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList,
+		"List of etcd servers to connect with (scheme://ip:port), comma separated.")
+
+	fs.StringVar(&s.StorageConfig.Prefix, "etcd-prefix", s.StorageConfig.Prefix,
+		"The prefix to prepend to all resource paths in etcd.")
+
+	fs.StringVar(&s.StorageConfig.Transport.KeyFile, "etcd-keyfile", s.StorageConfig.Transport.KeyFile,
+		"SSL key file used to secure etcd communication.")
+
+	fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile,
+		"SSL certification file used to secure etcd communication.")
+
+	fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile,
+		"SSL Certificate Authority file used to secure etcd communication.")
+
+	fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
+		"The file containing configuration for encryption providers to be used for storing secrets in etcd")
+
+	fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
+		"Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
+			"Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
+
+	fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
+		"The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
+
+	fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+
+		"Frequency of polling etcd for number of resources per type. 0 disables the metric collection.")
+
+	fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
+		"The interval of requests to poll etcd and update metric. 0 disables the metric collection")
+
+	fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
+		"The timeout to use when checking etcd health.")
+
+	fs.DurationVar(&s.StorageConfig.ReadycheckTimeout, "etcd-readycheck-timeout", s.StorageConfig.ReadycheckTimeout,
+		"The timeout to use when checking etcd readiness")
+
+	fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds,
+		"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
+}
+
+// ApplyTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
+func (s *EtcdOptions) ApplyTo(c *server.Config) error {
+	if s == nil {
+		return nil
+	}
+
+	storageConfigCopy := s.StorageConfig
+	if storageConfigCopy.StorageObjectCountTracker == nil {
+		storageConfigCopy.StorageObjectCountTracker = c.StorageObjectCountTracker
+	}
+
+	return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: storageConfigCopy}, c)
+}
+
+// ApplyWithStorageFactoryTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
+func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
+	if s == nil {
+		return nil
+	}
+
+	if !s.SkipHealthEndpoints {
+		if err := s.addEtcdHealthEndpoint(c); err != nil {
+			return err
+		}
+	}
+
+	// setup encryption
+	if err := s.maybeApplyResourceTransformers(c); err != nil {
+		return err
+	}
+
+	metrics.SetStorageMonitorGetter(monitorGetter(factory))
+
+	c.RESTOptionsGetter = s.CreateRESTOptionsGetter(factory, c.ResourceTransformers)
+	return nil
+}
+
+func monitorGetter(factory serverstorage.StorageFactory) func() (monitors []metrics.Monitor, err error) {
+	return func() (monitors []metrics.Monitor, err error) {
+		defer func() {
+			if err != nil {
+				for _, m := range monitors {
+					m.Close()
+				}
+			}
+		}()
+
+		var m metrics.Monitor
+		for _, cfg := range factory.Configs() {
+			m, err = storagefactory.CreateMonitor(cfg)
+			if err != nil {
+				return nil, err
+			}
+			monitors = append(monitors, m)
+		}
+		return monitors, nil
+	}
+}
+
+func (s *EtcdOptions) CreateRESTOptionsGetter(factory serverstorage.StorageFactory, resourceTransformers storagevalue.ResourceTransformers) generic.RESTOptionsGetter {
+	if resourceTransformers != nil {
+		factory = &transformerStorageFactory{
+			delegate:             factory,
+			resourceTransformers: resourceTransformers,
+		}
+	}
+	return &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
+}
+
+func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err error) {
+	if c.ResourceTransformers != nil {
+		return nil
+	}
+	if len(s.EncryptionProviderConfigFilepath) == 0 {
+		return nil
+	}
+
+	ctxServer := wait.ContextForChannel(c.DrainedNotify())
+	ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
+	defer func() {
+		// in case of error, we want to close partially initialized (if any) transformers
+		if err != nil {
+			closeTransformers()
+		}
+	}()
+
+	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
+	if err != nil {
+		return err
+	}
+
+	if s.EncryptionProviderConfigAutomaticReload {
+		// with reload=true we will always have 1 health check
+		if len(encryptionConfiguration.HealthChecks) != 1 {
+			return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
+		}
+
+		// Here the dynamic transformers take ownership of the transformers and their cancellation.
+		dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
+
+		// add post start hook to start hot reload controller
+		// adding this hook here will ensure that it gets configured exactly once
+		err = c.AddPostStartHook(
+			"start-encryption-provider-config-automatic-reload",
+			func(_ server.PostStartHookContext) error {
+				dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration(
+					"encryption-provider-config-automatic-reload-controller",
+					s.EncryptionProviderConfigFilepath,
+					dynamicTransformers,
+					encryptionConfiguration.EncryptionFileContentHash,
+					c.APIServerID,
+				)
+
+				go dynamicEncryptionConfigController.Run(ctxServer)
+
+				return nil
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
+		}
+
+		c.ResourceTransformers = dynamicTransformers
+		if !s.SkipHealthEndpoints {
+			addHealthChecksWithoutLivez(c, dynamicTransformers)
+		}
+	} else {
+		c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
+		if !s.SkipHealthEndpoints {
+			addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...)
+		}
+	}
+
+	return nil
+}
+
+func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) {
+	c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
+	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
+}
+
+func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
+	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
+	if err != nil {
+		return err
+	}
+	c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
+		return healthCheck()
+	}))
+
+	readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify())
+	if err != nil {
+		return err
+	}
+	c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error {
+		return readyCheck()
+	}))
+
+	return nil
+}
+
+type StorageFactoryRestOptionsFactory struct {
+	Options        EtcdOptions
+	StorageFactory serverstorage.StorageFactory
+}
+
+func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
+	storageConfig, err := f.StorageFactory.NewConfig(resource, example)
+	if err != nil {
+		return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
+	}
+
+	ret := generic.RESTOptions{
+		StorageConfig:             storageConfig,
+		Decorator:                 generic.UndecoratedStorage,
+		DeleteCollectionWorkers:   f.Options.DeleteCollectionWorkers,
+		EnableGarbageCollection:   f.Options.EnableGarbageCollection,
+		ResourcePrefix:            f.StorageFactory.ResourcePrefix(resource),
+		CountMetricPollPeriod:     f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
+	}
+
+	if ret.StorageObjectCountTracker == nil {
+		ret.StorageObjectCountTracker = storageConfig.StorageObjectCountTracker
+	}
+
+	if f.Options.EnableWatchCache {
+		sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
+		if err != nil {
+			return generic.RESTOptions{}, err
+		}
+		size, ok := sizes[resource]
+		if ok && size > 0 {
+			klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
+		}
+		if ok && size <= 0 {
+			klog.V(3).InfoS("Not using watch cache", "resource", resource)
+			ret.Decorator = generic.UndecoratedStorage
+		} else {
+			klog.V(3).InfoS("Using watch cache", "resource", resource)
+			ret.Decorator = genericregistry.StorageWithCacher()
+		}
+	}
+
+	return ret, nil
+}
+
+// ParseWatchCacheSizes turns a list of cache size values into a map of group resources
+// to requested sizes.
+func ParseWatchCacheSizes(cacheSizes []string) (map[schema.GroupResource]int, error) {
+	watchCacheSizes := make(map[schema.GroupResource]int)
+	for _, c := range cacheSizes {
+		tokens := strings.Split(c, "#")
+		if len(tokens) != 2 {
+			return nil, fmt.Errorf("invalid value of watch cache size: %s", c)
+		}
+
+		size, err := strconv.Atoi(tokens[1])
+		if err != nil {
+			return nil, fmt.Errorf("invalid size of watch cache size: %s", c)
+		}
+		if size < 0 {
+			return nil, fmt.Errorf("watch cache size cannot be negative: %s", c)
+		}
+		watchCacheSizes[schema.ParseGroupResource(tokens[0])] = size
+	}
+	return watchCacheSizes, nil
+}
+
+// WriteWatchCacheSizes turns a map of cache size values into a list of string specifications.
+func WriteWatchCacheSizes(watchCacheSizes map[schema.GroupResource]int) ([]string, error) {
+	var cacheSizes []string
+
+	for resource, size := range watchCacheSizes {
+		if size < 0 {
+			return nil, fmt.Errorf("watch cache size cannot be negative for resource %s", resource)
+		}
+		cacheSizes = append(cacheSizes, fmt.Sprintf("%s#%d", resource.String(), size))
+	}
+	return cacheSizes, nil
+}
+
+var _ serverstorage.StorageFactory = &SimpleStorageFactory{}
+
+// SimpleStorageFactory provides a StorageFactory implementation that should be used when different
+// resources essentially share the same storage config (as defined by the given storagebackend.Config).
+// It assumes the resources are stored at a path that is purely based on the schema.GroupResource.
+// Users that need flexibility and per resource overrides should use DefaultStorageFactory instead.
+type SimpleStorageFactory struct {
+	StorageConfig storagebackend.Config
+}
+
+func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
+	return s.StorageConfig.ForResource(resource), nil
+}
+
+func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
+	return resource.Group + "/" + resource.Resource
+}
+
+func (s *SimpleStorageFactory) Configs() []storagebackend.Config {
+	return serverstorage.Configs(s.StorageConfig)
+}
+
+func (s *SimpleStorageFactory) Backends() []serverstorage.Backend {
+	// nothing should ever call this method but we still provide a functional implementation
+	return serverstorage.Backends(s.StorageConfig)
+}
+
+var _ serverstorage.StorageFactory = &transformerStorageFactory{}
+
+type transformerStorageFactory struct {
+	delegate             serverstorage.StorageFactory
+	resourceTransformers storagevalue.ResourceTransformers
+}
+
+func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
+	config, err := t.delegate.NewConfig(resource, example)
+	if err != nil {
+		return nil, err
+	}
+
+	configCopy := *config
+	resourceConfig := configCopy.Config
+	resourceConfig.Transformer = t.resourceTransformers.TransformerForResource(resource)
+	configCopy.Config = resourceConfig
+
+	return &configCopy, nil
+}
+
+func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
+	return t.delegate.ResourcePrefix(resource)
+}
+
+func (t *transformerStorageFactory) Configs() []storagebackend.Config {
+	return t.delegate.Configs()
+}
+
+func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
+	return t.delegate.Backends()
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/server"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+)
+
+type FeatureOptions struct {
+	EnableProfiling           bool
+	DebugSocketPath           string
+	EnableContentionProfiling bool
+	EnablePriorityAndFairness bool
+}
+
+func NewFeatureOptions() *FeatureOptions {
+	defaults := server.NewConfig(serializer.CodecFactory{})
+
+	return &FeatureOptions{
+		EnableProfiling:           defaults.EnableProfiling,
+		DebugSocketPath:           defaults.DebugSocketPath,
+		EnableContentionProfiling: defaults.EnableContentionProfiling,
+		EnablePriorityAndFairness: true,
+	}
+}
+
+func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling,
+		"Enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling,
+		"Enable block profiling, if profiling is enabled")
+	fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath,
+		"Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path")
+	fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+
+		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
+}
+
+func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error {
+	if o == nil {
+		return nil
+	}
+
+	c.EnableProfiling = o.EnableProfiling
+	c.DebugSocketPath = o.DebugSocketPath
+	c.EnableContentionProfiling = o.EnableContentionProfiling
+
+	if o.EnablePriorityAndFairness {
+		if clientset == nil {
+			return fmt.Errorf("invalid configuration: priority and fairness requires a core Kubernetes client")
+		}
+		if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
+			return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
+
+		}
+		c.FlowControl = utilflowcontrol.New(
+			informers,
+			clientset.FlowcontrolV1(),
+			c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight,
+		)
+	}
+
+	return nil
+}
+
+func (o *FeatureOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	errs := []error{}
+	return errs
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/component-base/featuregate"
+)
+
+// RecommendedOptions contains the recommended options for running an API server.
+// If you add something to this list, it should be in a logical grouping.
+// Each of them can be nil to leave the feature unconfigured on ApplyTo.
+type RecommendedOptions struct {
+	Etcd           *EtcdOptions
+	SecureServing  *SecureServingOptionsWithLoopback
+	Authentication *DelegatingAuthenticationOptions
+	Authorization  *DelegatingAuthorizationOptions
+	Audit          *AuditOptions
+	Features       *FeatureOptions
+	CoreAPI        *CoreAPIOptions
+
+	// FeatureGate is a way to plumb feature gate through if you have them.
+	FeatureGate featuregate.FeatureGate
+	// ExtraAdmissionInitializers is called once after all ApplyTo from the options above, to pass the returned
+	// admission plugin initializers to Admission.ApplyTo.
+	ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
+	Admission                  *AdmissionOptions
+	// API Server Egress Selector is used to control outbound traffic from the API Server
+	EgressSelector *EgressSelectorOptions
+	// Traces contains options to control distributed request tracing.
+	Traces *TracingOptions
+}
+
+func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
+	sso := NewSecureServingOptions()
+
+	// We are composing recommended options for an aggregated api-server,
+	// whose client is typically a proxy multiplexing many operations ---
+	// notably including long-running ones --- into one HTTP/2 connection
+	// into this server.  So allow many concurrent operations.
+	sso.HTTP2MaxStreamsPerConnection = 1000
+
+	return &RecommendedOptions{
+		Etcd:           NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)),
+		SecureServing:  sso.WithLoopback(),
+		Authentication: NewDelegatingAuthenticationOptions(),
+		Authorization:  NewDelegatingAuthorizationOptions(),
+		Audit:          NewAuditOptions(),
+		Features:       NewFeatureOptions(),
+		CoreAPI:        NewCoreAPIOptions(),
+		// Wired a global by default that sadly people will abuse to have different meanings in different repos.
+		// Please consider creating your own FeatureGate so you can have a consistent meaning for what a variable contains
+		// across different repos.  Future you will thank you.
+		FeatureGate:                feature.DefaultFeatureGate,
+		ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
+		Admission:                  NewAdmissionOptions(),
+		EgressSelector:             NewEgressSelectorOptions(),
+		Traces:                     NewTracingOptions(),
+	}
+}
+
+func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
+	o.Etcd.AddFlags(fs)
+	o.SecureServing.AddFlags(fs)
+	o.Authentication.AddFlags(fs)
+	o.Authorization.AddFlags(fs)
+	o.Audit.AddFlags(fs)
+	o.Features.AddFlags(fs)
+	o.CoreAPI.AddFlags(fs)
+	o.Admission.AddFlags(fs)
+	o.EgressSelector.AddFlags(fs)
+	o.Traces.AddFlags(fs)
+}
+
+// ApplyTo adds RecommendedOptions to the server configuration.
+// pluginInitializers can be empty, it is only need for additional initializers.
+func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
+	if err := o.Etcd.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
+		return err
+	}
+	if err := o.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
+		return err
+	}
+	if err := o.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
+		return err
+	}
+	if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
+		return err
+	}
+	if err := o.Audit.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.CoreAPI.ApplyTo(config); err != nil {
+		return err
+	}
+	var kubeClient kubernetes.Interface
+	var dynamicClient dynamic.Interface
+	if config.ClientConfig != nil {
+		var err error
+		kubeClient, err = kubernetes.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
+		dynamicClient, err = dynamic.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
+	}
+	if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
+		return err
+	}
+	initializers, err := o.ExtraAdmissionInitializers(config)
+	if err != nil {
+		return err
+	}
+	if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, kubeClient, dynamicClient, o.FeatureGate,
+		initializers...); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (o *RecommendedOptions) Validate() []error {
+	errors := []error{}
+	errors = append(errors, o.Etcd.Validate()...)
+	errors = append(errors, o.SecureServing.Validate()...)
+	errors = append(errors, o.Authentication.Validate()...)
+	errors = append(errors, o.Authorization.Validate()...)
+	errors = append(errors, o.Audit.Validate()...)
+	errors = append(errors, o.Features.Validate()...)
+	errors = append(errors, o.CoreAPI.Validate()...)
+	errors = append(errors, o.Admission.Validate()...)
+	errors = append(errors, o.EgressSelector.Validate()...)
+	errors = append(errors, o.Traces.Validate()...)
+
+	return errors
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -0,0 +1,384 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/server"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/component-base/featuregate"
+	utilversion "k8s.io/component-base/version"
+
+	"github.com/spf13/pflag"
+)
+
+const (
+	corsAllowedOriginsHelpText = "List of allowed origins for CORS, comma separated. " +
+		"An allowed origin can be a regular expression to support subdomain matching. " +
+		"If this list is empty CORS will not be enabled. " +
+		"Please ensure each expression matches the entire hostname by anchoring " +
+		"to the start with '^' or including the '//' prefix, and by anchoring to the " +
+		"end with '$' or including the ':' port separator suffix. " +
+		"Examples of valid expressions are '//example\\.com(:|$)' and '^https://example\\.com(:|$)'"
+)
+
+// ServerRunOptions contains the options while running a generic api server.
+type ServerRunOptions struct {
+	AdvertiseAddress net.IP
+
+	CorsAllowedOriginList        []string
+	HSTSDirectives               []string
+	ExternalHost                 string
+	MaxRequestsInFlight          int
+	MaxMutatingRequestsInFlight  int
+	RequestTimeout               time.Duration
+	GoawayChance                 float64
+	LivezGracePeriod             time.Duration
+	MinRequestTimeout            int
+	StorageInitializationTimeout time.Duration
+	ShutdownDelayDuration        time.Duration
+	// We intentionally did not add a flag for this option. Users of the
+	// apiserver library can wire it to a flag.
+	JSONPatchMaxCopyBytes int64
+	// The limit on the request body size that would be accepted and
+	// decoded in a write request. 0 means no limit.
+	// We intentionally did not add a flag for this option. Users of the
+	// apiserver library can wire it to a flag.
+	MaxRequestBodyBytes int64
+
+	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
+	// Server during the graceful termination of the apiserver. If true, we wait
+	// for non longrunning requests in flight to be drained and then initiate a
+	// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
+	// Server as soon as ShutdownDelayDuration has elapsed.
+	// If enabled, after ShutdownDelayDuration elapses, any incoming request is
+	// rejected with a 429 status code and a 'Retry-After' response.
+	ShutdownSendRetryAfter bool
+
+	// ShutdownWatchTerminationGracePeriod, if set to a positive value,
+	// is the maximum duration the apiserver will wait for all active
+	// watch request(s) to drain.
+	// Once this grace period elapses, the apiserver will no longer
+	// wait for any active watch request(s) in flight to drain, it will
+	// proceed to the next step in the graceful server shutdown process.
+	// If set to a positive value, the apiserver will keep track of the
+	// number of active watch request(s) in flight and during shutdown
+	// it will wait, at most, for the specified duration and allow these
+	// active watch requests to drain with some rate limiting in effect.
+	// The default is zero, which implies the apiserver will not keep
+	// track of active watch request(s) in flight and will not wait
+	// for them to drain, this maintains backward compatibility.
+	// This grace period is orthogonal to other grace periods, and
+	// it is not overridden by any other grace period.
+	ShutdownWatchTerminationGracePeriod time.Duration
+
+	// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
+	ComponentGlobalsRegistry featuregate.ComponentGlobalsRegistry
+	// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
+	ComponentName string
+}
+
+func NewServerRunOptions() *ServerRunOptions {
+	if featuregate.DefaultComponentGlobalsRegistry.EffectiveVersionFor(featuregate.DefaultKubeComponent) == nil {
+		featureGate := utilfeature.DefaultMutableFeatureGate
+		effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+		utilruntime.Must(featuregate.DefaultComponentGlobalsRegistry.Register(featuregate.DefaultKubeComponent, effectiveVersion, featureGate))
+	}
+
+	return NewServerRunOptionsForComponent(featuregate.DefaultKubeComponent, featuregate.DefaultComponentGlobalsRegistry)
+}
+
+func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry featuregate.ComponentGlobalsRegistry) *ServerRunOptions {
+	defaults := server.NewConfig(serializer.CodecFactory{})
+	return &ServerRunOptions{
+		MaxRequestsInFlight:                 defaults.MaxRequestsInFlight,
+		MaxMutatingRequestsInFlight:         defaults.MaxMutatingRequestsInFlight,
+		RequestTimeout:                      defaults.RequestTimeout,
+		LivezGracePeriod:                    defaults.LivezGracePeriod,
+		MinRequestTimeout:                   defaults.MinRequestTimeout,
+		StorageInitializationTimeout:        defaults.StorageInitializationTimeout,
+		ShutdownDelayDuration:               defaults.ShutdownDelayDuration,
+		ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
+		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
+		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
+		ShutdownSendRetryAfter:              false,
+		ComponentName:                       componentName,
+		ComponentGlobalsRegistry:            componentGlobalsRegistry,
+	}
+}
+
+// ApplyTo applies the run options to the method receiver and returns self
+func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
+	if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
+		return err
+	}
+	c.CorsAllowedOriginList = s.CorsAllowedOriginList
+	c.HSTSDirectives = s.HSTSDirectives
+	c.ExternalAddress = s.ExternalHost
+	c.MaxRequestsInFlight = s.MaxRequestsInFlight
+	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
+	c.LivezGracePeriod = s.LivezGracePeriod
+	c.RequestTimeout = s.RequestTimeout
+	c.GoawayChance = s.GoawayChance
+	c.MinRequestTimeout = s.MinRequestTimeout
+	c.StorageInitializationTimeout = s.StorageInitializationTimeout
+	c.ShutdownDelayDuration = s.ShutdownDelayDuration
+	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
+	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
+	c.PublicAddress = s.AdvertiseAddress
+	c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
+	c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
+	c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
+	c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
+
+	return nil
+}
+
+// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
+func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
+	if secure == nil {
+		return nil
+	}
+
+	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
+		hostIP, err := secure.DefaultExternalAddress()
+		if err != nil {
+			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
+				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
+		}
+		s.AdvertiseAddress = hostIP
+	}
+
+	return nil
+}
+
+// Validate checks validation of ServerRunOptions
+func (s *ServerRunOptions) Validate() []error {
+	errors := []error{}
+
+	if s.LivezGracePeriod < 0 {
+		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
+	}
+
+	if s.MaxRequestsInFlight < 0 {
+		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
+	}
+	if s.MaxMutatingRequestsInFlight < 0 {
+		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
+	}
+
+	if s.RequestTimeout.Nanoseconds() < 0 {
+		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
+	}
+
+	if s.GoawayChance < 0 || s.GoawayChance > 0.02 {
+		errors = append(errors, fmt.Errorf("--goaway-chance can not be less than 0 or greater than 0.02"))
+	}
+
+	if s.MinRequestTimeout < 0 {
+		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
+	}
+
+	if s.StorageInitializationTimeout < 0 {
+		errors = append(errors, fmt.Errorf("--storage-initialization-timeout can not be negative value"))
+	}
+
+	if s.ShutdownDelayDuration < 0 {
+		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
+	}
+
+	if s.ShutdownWatchTerminationGracePeriod < 0 {
+		errors = append(errors, fmt.Errorf("shutdown-watch-termination-grace-period, if provided, can not be a negative value"))
+	}
+
+	if s.JSONPatchMaxCopyBytes < 0 {
+		errors = append(errors, fmt.Errorf("ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value"))
+	}
+
+	if s.MaxRequestBodyBytes < 0 {
+		errors = append(errors, fmt.Errorf("ServerRunOptions.MaxRequestBodyBytes can not be negative value"))
+	}
+
+	if err := validateHSTSDirectives(s.HSTSDirectives); err != nil {
+		errors = append(errors, err)
+	}
+
+	if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
+		errors = append(errors, err)
+	}
+	if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
+		errors = append(errors, errs...)
+	}
+	return errors
+}
+
+func validateHSTSDirectives(hstsDirectives []string) error {
+	// HSTS Headers format: Strict-Transport-Security:max-age=expireTime [;includeSubDomains] [;preload]
+	// See https://tools.ietf.org/html/rfc6797#section-6.1 for more information
+	allErrors := []error{}
+	for _, hstsDirective := range hstsDirectives {
+		if len(strings.TrimSpace(hstsDirective)) == 0 {
+			allErrors = append(allErrors, fmt.Errorf("empty value in strict-transport-security-directives"))
+			continue
+		}
+		if hstsDirective != "includeSubDomains" && hstsDirective != "preload" {
+			maxAgeDirective := strings.Split(hstsDirective, "=")
+			if len(maxAgeDirective) != 2 || maxAgeDirective[0] != "max-age" {
+				allErrors = append(allErrors, fmt.Errorf("--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information"))
+			}
+		}
+	}
+	return errors.NewAggregate(allErrors)
+}
+
+func validateCorsAllowedOriginList(corsAllowedOriginList []string) error {
+	allErrors := []error{}
+	validateRegexFn := func(regexpStr string) error {
+		if _, err := regexp.Compile(regexpStr); err != nil {
+			return err
+		}
+
+		// the regular expression should pin to the start and end of the host
+		// in the origin header, this will prevent CVE-2022-1996.
+		// possible ways it can pin to the start of host in the origin header:
+		//   - match the start of the origin with '^'
+		//   - match what separates the scheme and host with '//' or '://',
+		//     this pins to the start of host in the origin header.
+		// possible ways it can match the end of the host in the origin header:
+		//   - match the end of the origin with '$'
+		//   - with a capture group that matches the host and port separator '(:|$)'
+		// We will relax the validation to check if these regex markers
+		// are present in the user specified expression.
+		var pinStart, pinEnd bool
+		for _, prefix := range []string{"^", "//"} {
+			if strings.Contains(regexpStr, prefix) {
+				pinStart = true
+				break
+			}
+		}
+		for _, suffix := range []string{"$", ":"} {
+			if strings.Contains(regexpStr, suffix) {
+				pinEnd = true
+				break
+			}
+		}
+		if !pinStart || !pinEnd {
+			return fmt.Errorf("regular expression does not pin to start/end of host in the origin header")
+		}
+		return nil
+	}
+
+	for _, regexp := range corsAllowedOriginList {
+		if len(regexp) == 0 {
+			allErrors = append(allErrors, fmt.Errorf("empty value in --cors-allowed-origins, help: %s", corsAllowedOriginsHelpText))
+			continue
+		}
+
+		if err := validateRegexFn(regexp); err != nil {
+			err = fmt.Errorf("--cors-allowed-origins has an invalid regular expression: %v, help: %s", err, corsAllowedOriginsHelpText)
+			allErrors = append(allErrors, err)
+		}
+	}
+	return errors.NewAggregate(allErrors)
+}
+
+// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
+func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
+	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
+	// arrange these text blocks sensibly. Grrr.
+
+	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
+		"The IP address on which to advertise the apiserver to members of the cluster. This "+
+		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
+		"will be used. If --bind-address is unspecified, the host's default interface will "+
+		"be used.")
+
+	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, corsAllowedOriginsHelpText)
+
+	fs.StringSliceVar(&s.HSTSDirectives, "strict-transport-security-directives", s.HSTSDirectives, ""+
+		"List of directives for HSTS, comma separated. If this list is empty, then HSTS directives will not "+
+		"be added. Example: 'max-age=31536000,includeSubDomains,preload'")
+
+	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
+		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
+
+	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
+		"This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of non-mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
+
+	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
+		"This and --max-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
+
+	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
+		"An optional field indicating the duration a handler must keep a request open before timing "+
+		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
+		"--min-request-timeout for specific types of requests.")
+
+	fs.Float64Var(&s.GoawayChance, "goaway-chance", s.GoawayChance, ""+
+		"To prevent HTTP/2 clients from getting stuck on a single apiserver, randomly close a connection (GOAWAY). "+
+		"The client's other in-flight requests won't be affected, and the client will reconnect, likely landing on a different apiserver after going through the load balancer again. "+
+		"This argument sets the fraction of requests that will be sent a GOAWAY. Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. "+
+		"Min is 0 (off), Max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point.")
+
+	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
+		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
+		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
+		"that unfinished post-start hooks will complete successfully and therefore return true.")
+
+	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
+		"An optional field indicating the minimum number of seconds a handler must keep "+
+		"a request open before timing it out. Currently only honored by the watch request "+
+		"handler, which picks a randomized value above this number as the connection timeout, "+
+		"to spread out load.")
+
+	fs.DurationVar(&s.StorageInitializationTimeout, "storage-initialization-timeout", s.StorageInitializationTimeout,
+		"Maximum amount of time to wait for storage initialization before declaring apiserver ready. Defaults to 1m.")
+
+	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
+		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
+		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
+		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
+
+	fs.BoolVar(&s.ShutdownSendRetryAfter, "shutdown-send-retry-after", s.ShutdownSendRetryAfter, ""+
+		"If true the HTTP Server will continue listening until all non long running request(s) in flight have been drained, "+
+		"during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
+		"in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
+
+	fs.DurationVar(&s.ShutdownWatchTerminationGracePeriod, "shutdown-watch-termination-grace-period", s.ShutdownWatchTerminationGracePeriod, ""+
+		"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
+		"for active watch request(s) to drain during the graceful server shutdown window.")
+
+	s.ComponentGlobalsRegistry.AddFlags(fs)
+}
+
+// Complete fills missing fields with defaults.
+func (s *ServerRunOptions) Complete() error {
+	return s.ComponentGlobalsRegistry.SetFallback()
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -0,0 +1,422 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"path"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+type SecureServingOptions struct {
+	BindAddress net.IP
+	// BindPort is ignored when Listener is set, will serve https even with 0.
+	BindPort int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+	// DisableHTTP2Serving indicates that http2 serving should not be enabled.
+	DisableHTTP2Serving bool
+	// Required set to true means that BindPort cannot be zero.
+	Required bool
+	// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this
+	// is set to BindAddress if the later no loopback, or to the first host interface address.
+	ExternalAddress net.IP
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
+
+	// ServerCert is the TLS cert info for serving secure traffic
+	ServerCert GeneratableKeyCert
+	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
+	SNICertKeys []cliflag.NamedCertKey
+	// CipherSuites is the list of allowed cipher suites for the server.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	CipherSuites []string
+	// MinTLSVersion is the minimum TLS version supported.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	MinTLSVersion string
+
+	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
+	// A value of zero means to use the default provided by golang's HTTP/2 support.
+	HTTP2MaxStreamsPerConnection int
+
+	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
+	// more than one instance to bind on the same address and port.
+	PermitPortSharing bool
+
+	// PermitAddressSharing controls if SO_REUSEADDR is used when binding the port.
+	PermitAddressSharing bool
+}
+
+type CertKey struct {
+	// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain
+	CertFile string
+	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
+	KeyFile string
+}
+
+type GeneratableKeyCert struct {
+	// CertKey allows setting an explicit cert/key file to use.
+	CertKey CertKey
+
+	// CertDirectory specifies a directory to write generated certificates to if CertFile/KeyFile aren't explicitly set.
+	// PairName is used to determine the filenames within CertDirectory.
+	// If CertDirectory and PairName are not set, an in-memory certificate will be generated.
+	CertDirectory string
+	// PairName is the name which will be used with CertDirectory to make a cert and key filenames.
+	// It becomes CertDirectory/PairName.crt and CertDirectory/PairName.key
+	PairName string
+
+	// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set.
+	GeneratedCert dynamiccertificates.CertKeyContentProvider
+
+	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
+	// The format is:
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
+	FixtureDirectory string
+}
+
+func NewSecureServingOptions() *SecureServingOptions {
+	return &SecureServingOptions{
+		BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
+		BindPort:    443,
+		ServerCert: GeneratableKeyCert{
+			PairName:      "apiserver",
+			CertDirectory: "apiserver.local.config/certificates",
+		},
+	}
+}
+
+func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) {
+	if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() {
+		return s.ExternalAddress, nil
+	}
+	return utilnet.ResolveBindAddress(s.BindAddress)
+}
+
+func (s *SecureServingOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	errors := []error{}
+
+	if s.Required && s.BindPort < 1 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("--secure-port %v must be between 1 and 65535, inclusive. It cannot be turned off with 0", s.BindPort))
+	} else if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort))
+	}
+
+	if (len(s.ServerCert.CertKey.CertFile) != 0 || len(s.ServerCert.CertKey.KeyFile) != 0) && s.ServerCert.GeneratedCert != nil {
+		errors = append(errors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set"))
+	}
+
+	return errors
+}
+
+func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
+		"The IP address on which to listen for the --secure-port port. The "+
+		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
+		"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces and IP address families will be used.")
+
+	desc := "The port on which to serve HTTPS with authentication and authorization."
+	if s.Required {
+		desc += " It cannot be switched off with 0."
+	} else {
+		desc += " If 0, don't serve HTTPS at all."
+	}
+	fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc)
+
+	fs.BoolVar(&s.DisableHTTP2Serving, "disable-http2-serving", s.DisableHTTP2Serving,
+		"If true, HTTP2 serving will be disabled [default=false]")
+
+	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
+		"The directory where the TLS certs are located. "+
+		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
+
+	fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
+		"--tls-private-key-file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to the directory specified by --cert-dir.")
+
+	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
+		"File containing the default x509 private key matching --tls-cert-file.")
+
+	tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
+	tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
+	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
+		"Comma-separated list of cipher suites for the server. "+
+			"If omitted, the default Go cipher suites will be used. \n"+
+			"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
+			"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
+
+	tlsPossibleVersions := cliflag.TLSPossibleVersions()
+	fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
+		"Minimum TLS version supported. "+
+			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
+
+	fs.Var(cliflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
+		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
+		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
+		"segments. The domain patterns also allow IP addresses, but IPs should only be used if "+
+		"the apiserver has visibility to the IP address requested by a client. "+
+		"If no domain patterns are provided, the names of the certificate are "+
+		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
+		"trump over extracted names. For multiple key/certificate pairs, use the "+
+		"--tls-sni-cert-key multiple times. "+
+		"Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".")
+
+	fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+
+		"The limit that the server gives to clients for "+
+		"the maximum number of streams in an HTTP/2 connection. "+
+		"Zero means to use golang's default.")
+
+	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
+		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
+			"more than one instance to bind on the same address and port. [default=false]")
+
+	fs.BoolVar(&s.PermitAddressSharing, "permit-address-sharing", s.PermitAddressSharing,
+		"If true, SO_REUSEADDR will be used when binding the port. This allows binding "+
+			"to wildcard IPs like 0.0.0.0 and specific IPs in parallel, and it avoids waiting "+
+			"for the kernel to release sockets in TIME_WAIT state. [default=false]")
+}
+
+// ApplyTo fills up serving information in the server configuration.
+func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error {
+	if s == nil {
+		return nil
+	}
+	if s.BindPort <= 0 && s.Listener == nil {
+		return nil
+	}
+
+	if s.Listener == nil {
+		var err error
+		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
+
+		c := net.ListenConfig{}
+
+		ctls := multipleControls{}
+		if s.PermitPortSharing {
+			ctls = append(ctls, permitPortReuse)
+		}
+		if s.PermitAddressSharing {
+			ctls = append(ctls, permitAddressReuse)
+		}
+		if len(ctls) > 0 {
+			c.Control = ctls.Control
+		}
+
+		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	} else {
+		if _, ok := s.Listener.Addr().(*net.TCPAddr); !ok {
+			return fmt.Errorf("failed to parse ip and port from listener")
+		}
+		s.BindPort = s.Listener.Addr().(*net.TCPAddr).Port
+		s.BindAddress = s.Listener.Addr().(*net.TCPAddr).IP
+	}
+
+	*config = &server.SecureServingInfo{
+		Listener:                     s.Listener,
+		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
+		DisableHTTP2:                 s.DisableHTTP2Serving,
+	}
+	c := *config
+
+	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
+	// load main cert *original description until 2023-08-18*
+
+	/*
+		kubernetes mutual (2-way) x509 between client and apiserver:
+
+			>1. apiserver sending its apiserver certificate along with its publickey to client
+			2. client verifies the apiserver certificate sent against its cluster certificate authority data
+			3. client sending its client certificate along with its public key to the apiserver
+			4. apiserver verifies the client certificate sent against its cluster certificate authority data
+
+			description:
+				here, with this block,
+				apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo
+				for client to later in the step 2 verify the apiserver certificate during the handshake
+				when making a request
+
+			normal args related to this stage:
+				--tls-cert-file string  File containing the default x509 Certificate for HTTPS.
+					(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and
+					--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate
+					and key are generated for the public address and saved to the directory specified by
+					--cert-dir
+				--tls-private-key-file string  File containing the default x509 private key matching --tls-cert-file.
+
+				(retrievable from "kube-apiserver --help" command)
+				(suggested by @deads2k)
+
+			see also:
+				- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
+	*/
+
+	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
+		var err error
+		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
+		if err != nil {
+			return err
+		}
+	} else if s.ServerCert.GeneratedCert != nil {
+		c.Cert = s.ServerCert.GeneratedCert
+	}
+
+	if len(s.CipherSuites) != 0 {
+		cipherSuites, err := cliflag.TLSCipherSuites(s.CipherSuites)
+		if err != nil {
+			return err
+		}
+		c.CipherSuites = cipherSuites
+	}
+
+	var err error
+	c.MinTLSVersion, err = cliflag.TLSVersion(s.MinTLSVersion)
+	if err != nil {
+		return err
+	}
+
+	// load SNI certs
+	namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys))
+	for _, nck := range s.SNICertKeys {
+		tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...)
+		namedTLSCerts = append(namedTLSCerts, tlsCert)
+		if err != nil {
+			return fmt.Errorf("failed to load SNI cert and key: %v", err)
+		}
+	}
+	c.SNICerts = namedTLSCerts
+
+	return nil
+}
+
+func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateDNS []string, alternateIPs []net.IP) error {
+	if s == nil || (s.BindPort == 0 && s.Listener == nil) {
+		return nil
+	}
+	keyCert := &s.ServerCert.CertKey
+	if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
+		return nil
+	}
+
+	canReadCertAndKey := false
+	if len(s.ServerCert.CertDirectory) > 0 {
+		if len(s.ServerCert.PairName) == 0 {
+			return fmt.Errorf("PairName is required if CertDirectory is set")
+		}
+		keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt")
+		keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key")
+		if canRead, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile); err != nil {
+			return err
+		} else {
+			canReadCertAndKey = canRead
+		}
+	}
+
+	if !canReadCertAndKey {
+		// add either the bind address or localhost to the valid alternates
+		if s.BindAddress.IsUnspecified() {
+			alternateDNS = append(alternateDNS, "localhost")
+		} else {
+			alternateIPs = append(alternateIPs, s.BindAddress)
+		}
+
+		if cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(publicAddress, alternateIPs, alternateDNS, s.ServerCert.FixtureDirectory); err != nil {
+			return fmt.Errorf("unable to generate self signed cert: %v", err)
+		} else if len(keyCert.CertFile) > 0 && len(keyCert.KeyFile) > 0 {
+			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
+				return err
+			}
+			if err := keyutil.WriteKey(keyCert.KeyFile, key); err != nil {
+				return err
+			}
+			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
+		} else {
+			s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Generated self-signed cert in-memory")
+		}
+	}
+
+	return nil
+}
+
+func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
+	if len(network) == 0 {
+		network = "tcp"
+	}
+
+	ln, err := config.Listen(context.TODO(), network, addr)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	return ln, tcpAddr.Port, nil
+}
+
+type multipleControls []func(network, addr string, conn syscall.RawConn) error
+
+func (mcs multipleControls) Control(network, addr string, conn syscall.RawConn) error {
+	for _, c := range mcs {
+		if err := c(network, addr, conn); err != nil {
+			return err
+		}
+	}
+	return nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -0,0 +1,44 @@
+//go:build !windows
+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog/v2"
+)
+
+func permitPortReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEPORT on socket: %v", err)
+		}
+	})
+}
+
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEADDR on socket: %v", err)
+		}
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -0,0 +1,35 @@
+//go:build windows
+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func permitPortReuse(network, address string, c syscall.RawConn) error {
+	return fmt.Errorf("port reuse is not supported on Windows")
+}
+
+// Windows supports SO_REUSEADDR, but it may cause undefined behavior, as
+// there is no protection against port hijacking.
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return fmt.Errorf("address reuse is not supported on Windows")
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
+)
+
+type SecureServingOptionsWithLoopback struct {
+	*SecureServingOptions
+}
+
+func (o *SecureServingOptions) WithLoopback() *SecureServingOptionsWithLoopback {
+	return &SecureServingOptionsWithLoopback{o}
+}
+
+// ApplyTo fills up serving information in the server configuration.
+func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config) error {
+	if s == nil || s.SecureServingOptions == nil || secureServingInfo == nil {
+		return nil
+	}
+
+	if err := s.SecureServingOptions.ApplyTo(secureServingInfo); err != nil {
+		return err
+	}
+
+	if *secureServingInfo == nil || loopbackClientConfig == nil {
+		return nil
+	}
+
+	// create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and
+	// let the server return it when the loopback client connects.
+	certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
+	}
+	certProvider, err := dynamiccertificates.NewStaticSNICertKeyContent("self-signed loopback", certPem, keyPem, server.LoopbackClientServerNameOverride)
+	if err != nil {
+		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
+	}
+
+	// Write to the front of SNICerts so that this overrides any other certs with the same name
+	(*secureServingInfo).SNICerts = append([]dynamiccertificates.SNICertKeyContentProvider{certProvider}, (*secureServingInfo).SNICerts...)
+
+	secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.New().String(), certPem)
+	switch {
+	// if we failed and there's no fallback loopback client config, we need to fail
+	case err != nil && *loopbackClientConfig == nil:
+		(*secureServingInfo).SNICerts = (*secureServingInfo).SNICerts[1:]
+		return err
+
+	// if we failed, but we already have a fallback loopback client config (usually insecure), allow it
+	case err != nil && *loopbackClientConfig != nil:
+
+	default:
+		*loopbackClientConfig = secureLoopbackClientConfig
+	}
+
+	return nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/options/tracing.go

@@ -0,0 +1,166 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+
+	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/metric/noop"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+	"google.golang.org/grpc"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/feature"
+	tracing "k8s.io/component-base/tracing"
+	tracingapi "k8s.io/component-base/tracing/api/v1"
+	"k8s.io/utils/path"
+)
+
+const apiserverService = "apiserver"
+
+var (
+	cfgScheme = runtime.NewScheme()
+	codecs    = serializer.NewCodecFactory(cfgScheme, serializer.EnableStrict)
+)
+
+func init() {
+	// Prevent memory leak from OTel metrics, which we don't use:
+	// https://github.com/open-telemetry/opentelemetry-go-contrib/issues/5190
+	otel.SetMeterProvider(noop.NewMeterProvider())
+}
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// TracingOptions contain configuration options for tracing
+// exporters
+type TracingOptions struct {
+	// ConfigFile is the file path with api-server tracing configuration.
+	ConfigFile string
+}
+
+// NewTracingOptions creates a new instance of TracingOptions
+func NewTracingOptions() *TracingOptions {
+	return &TracingOptions{}
+}
+
+// AddFlags adds flags related to tracing to the specified FlagSet
+func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "tracing-config-file", o.ConfigFile,
+		"File with apiserver tracing configuration.")
+}
+
+// ApplyTo fills up Tracing config with options.
+func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Config) error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+	if !feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		return fmt.Errorf("APIServerTracing feature is not enabled, but tracing config file was provided")
+	}
+
+	traceConfig, err := ReadTracingConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read tracing config: %v", err)
+	}
+
+	errs := tracingapi.ValidateTracingConfiguration(traceConfig, feature.DefaultFeatureGate, nil)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate tracing configuration: %v", errs.ToAggregate())
+	}
+
+	opts := []otlptracegrpc.Option{}
+	if es != nil {
+		// Only use the egressselector dialer if egressselector is enabled.
+		// Endpoint is on the "ControlPlane" network
+		egressDialer, err := es.Lookup(egressselector.ControlPlane.AsNetworkContext())
+		if err != nil {
+			return err
+		}
+		if egressDialer != nil {
+			otelDialer := func(ctx context.Context, addr string) (net.Conn, error) {
+				return egressDialer(ctx, "tcp", addr)
+			}
+			opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer)))
+		}
+	}
+
+	resourceOpts := []resource.Option{
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(apiserverService),
+			semconv.ServiceInstanceIDKey.String(c.APIServerID),
+		),
+	}
+	tp, err := tracing.NewProvider(context.Background(), traceConfig, opts, resourceOpts)
+	if err != nil {
+		return err
+	}
+	c.TracerProvider = tp
+	if c.LoopbackClientConfig != nil {
+		c.LoopbackClientConfig.Wrap(tracing.WrapperFor(c.TracerProvider))
+	}
+	return nil
+}
+
+// Validate verifies flags passed to TracingOptions.
+func (o *TracingOptions) Validate() (errs []error) {
+	if o == nil || o.ConfigFile == "" {
+		return
+	}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists {
+		errs = append(errs, fmt.Errorf("tracing-config-file %s does not exist", o.ConfigFile))
+	} else if err != nil {
+		errs = append(errs, fmt.Errorf("error checking if tracing-config-file %s exists: %v", o.ConfigFile, err))
+	}
+	return
+}
+
+// ReadTracingConfiguration reads the tracing configuration from a file
+func ReadTracingConfiguration(configFilePath string) (*tracingapi.TracingConfiguration, error) {
+	if configFilePath == "" {
+		return nil, fmt.Errorf("tracing config file was empty")
+	}
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read tracing configuration from %q: %v", configFilePath, err)
+	}
+	internalConfig := &apiserver.TracingConfiguration{}
+	// this handles json/yaml/whatever, and decodes all registered version to the internal version
+	if err := runtime.DecodeInto(codecs.UniversalDecoder(), data, internalConfig); err != nil {
+		return nil, fmt.Errorf("unable to decode tracing configuration data: %v", err)
+	}
+	return &internalConfig.TracingConfiguration, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/plugins.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+// This file exists to force the desired plugin implementations to be linked into genericapi pkg.
+import (
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
+	mutatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/mutating"
+	validatingadmissionpolicy "k8s.io/apiserver/pkg/admission/plugin/policy/validating"
+	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
+	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+)
+
+// RegisterAllAdmissionPlugins registers all admission plugins
+func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
+	lifecycle.Register(plugins)
+	validatingwebhook.Register(plugins)
+	mutatingwebhook.Register(plugins)
+	validatingadmissionpolicy.Register(plugins)
+	mutatingadmissionpolicy.Register(plugins)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/resourceconfig/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package resourceconfig contains the resource config related helper functions.
+package resourceconfig // import "k8s.io/apiserver/pkg/server/resourceconfig"

e2e/vendor/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resourceconfig
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	serverstore "k8s.io/apiserver/pkg/server/storage"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+// GroupVersionRegistry provides access to registered group versions.
+type GroupVersionRegistry interface {
+	// IsGroupRegistered returns true if given group is registered.
+	IsGroupRegistered(group string) bool
+	// IsVersionRegistered returns true if given version is registered.
+	IsVersionRegistered(v schema.GroupVersion) bool
+	// PrioritizedVersionsAllGroups returns all registered group versions.
+	PrioritizedVersionsAllGroups() []schema.GroupVersion
+}
+
+// MergeResourceEncodingConfigs merges the given defaultResourceConfig with specific GroupVersionResource overrides.
+func MergeResourceEncodingConfigs(
+	defaultResourceEncoding *serverstore.DefaultResourceEncodingConfig,
+	resourceEncodingOverrides []schema.GroupVersionResource,
+) *serverstore.DefaultResourceEncodingConfig {
+	resourceEncodingConfig := defaultResourceEncoding
+	for _, gvr := range resourceEncodingOverrides {
+		resourceEncodingConfig.SetResourceEncoding(gvr.GroupResource(), gvr.GroupVersion(),
+			schema.GroupVersion{Group: gvr.Group, Version: runtime.APIVersionInternal})
+	}
+	return resourceEncodingConfig
+}
+
+// Recognized values for the --runtime-config parameter to enable/disable groups of APIs
+const (
+	APIAll   = "api/all"
+	APIGA    = "api/ga"
+	APIBeta  = "api/beta"
+	APIAlpha = "api/alpha"
+)
+
+var (
+	gaPattern    = regexp.MustCompile(`^v\d+$`)
+	betaPattern  = regexp.MustCompile(`^v\d+beta\d+$`)
+	alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
+
+	groupVersionMatchers = map[string]func(gv schema.GroupVersion) bool{
+		// allows users to address all api versions
+		APIAll: func(gv schema.GroupVersion) bool { return true },
+		// allows users to address all api versions in the form v[0-9]+
+		APIGA: func(gv schema.GroupVersion) bool { return gaPattern.MatchString(gv.Version) },
+		// allows users to address all beta api versions
+		APIBeta: func(gv schema.GroupVersion) bool { return betaPattern.MatchString(gv.Version) },
+		// allows users to address all alpha api versions
+		APIAlpha: func(gv schema.GroupVersion) bool { return alphaPattern.MatchString(gv.Version) },
+	}
+
+	groupVersionMatchersOrder = []string{APIAll, APIGA, APIBeta, APIAlpha}
+)
+
+// MergeAPIResourceConfigs merges the given defaultAPIResourceConfig with the given resourceConfigOverrides.
+// Exclude the groups not registered in registry, and check if version is
+// not registered in group, then it will fail.
+func MergeAPIResourceConfigs(
+	defaultAPIResourceConfig *serverstore.ResourceConfig,
+	resourceConfigOverrides cliflag.ConfigurationMap,
+	registry GroupVersionRegistry,
+) (*serverstore.ResourceConfig, error) {
+	resourceConfig := defaultAPIResourceConfig
+	overrides := resourceConfigOverrides
+
+	for _, flag := range groupVersionMatchersOrder {
+		if value, ok := overrides[flag]; ok {
+			if value == "false" {
+				resourceConfig.DisableMatchingVersions(groupVersionMatchers[flag])
+			} else if value == "true" {
+				resourceConfig.EnableMatchingVersions(groupVersionMatchers[flag])
+			} else {
+				return nil, fmt.Errorf("invalid value %v=%v", flag, value)
+			}
+		}
+	}
+
+	type versionEnablementPreference struct {
+		key          string
+		enabled      bool
+		groupVersion schema.GroupVersion
+	}
+	type resourceEnablementPreference struct {
+		key                  string
+		enabled              bool
+		groupVersionResource schema.GroupVersionResource
+	}
+	versionPreferences := []versionEnablementPreference{}
+	resourcePreferences := []resourceEnablementPreference{}
+
+	// "<resourceSpecifier>={true|false} allows users to enable/disable API.
+	// This takes preference over api/all, if specified.
+	// Iterate through all group/version overrides specified in runtimeConfig.
+	for key := range overrides {
+		// Have already handled them above. Can skip them here.
+		if _, ok := groupVersionMatchers[key]; ok {
+			continue
+		}
+
+		tokens := strings.Split(key, "/")
+		if len(tokens) < 2 || len(tokens) > 3 {
+			continue
+		}
+		groupVersionString := tokens[0] + "/" + tokens[1]
+		groupVersion, err := schema.ParseGroupVersion(groupVersionString)
+		if err != nil {
+			return nil, fmt.Errorf("invalid key %s", key)
+		}
+
+		// Exclude group not registered into the registry.
+		if !registry.IsGroupRegistered(groupVersion.Group) {
+			continue
+		}
+
+		// Verify that the groupVersion is registered into registry.
+		if !registry.IsVersionRegistered(groupVersion) {
+			return nil, fmt.Errorf("group version %s that has not been registered", groupVersion.String())
+		}
+		enabled, err := getRuntimeConfigValue(overrides, key, false)
+		if err != nil {
+			return nil, err
+		}
+
+		switch len(tokens) {
+		case 2:
+			versionPreferences = append(versionPreferences, versionEnablementPreference{
+				key:          key,
+				enabled:      enabled,
+				groupVersion: groupVersion,
+			})
+		case 3:
+			if strings.ToLower(tokens[2]) != tokens[2] {
+				return nil, fmt.Errorf("invalid key %v: group/version/resource and resource is always lowercase plural, not %q", key, tokens[2])
+			}
+			resourcePreferences = append(resourcePreferences, resourceEnablementPreference{
+				key:                  key,
+				enabled:              enabled,
+				groupVersionResource: groupVersion.WithResource(tokens[2]),
+			})
+		}
+	}
+
+	// apply version preferences first, so that we can remove the hardcoded resource preferences that are being overridden
+	for _, versionPreference := range versionPreferences {
+		if versionPreference.enabled {
+			// enable the groupVersion for "group/version=true"
+			resourceConfig.EnableVersions(versionPreference.groupVersion)
+
+		} else {
+			// disable the groupVersion only for "group/version=false"
+			resourceConfig.DisableVersions(versionPreference.groupVersion)
+		}
+	}
+
+	// apply resource preferences last, so they have the highest priority
+	for _, resourcePreference := range resourcePreferences {
+		if resourcePreference.enabled {
+			// enable the resource for "group/version/resource=true"
+			resourceConfig.EnableResources(resourcePreference.groupVersionResource)
+		} else {
+			resourceConfig.DisableResources(resourcePreference.groupVersionResource)
+		}
+	}
+
+	return resourceConfig, nil
+}
+
+func getRuntimeConfigValue(overrides cliflag.ConfigurationMap, apiKey string, defaultValue bool) (bool, error) {
+	flagValue, ok := overrides[apiKey]
+	if ok {
+		if flagValue == "" {
+			return true, nil
+		}
+		boolValue, err := strconv.ParseBool(flagValue)
+		if err != nil {
+			return false, fmt.Errorf("invalid value of %s: %s, err: %v", apiKey, flagValue, err)
+		}
+		return boolValue, nil
+	}
+	return defaultValue, nil
+}
+
+// ParseGroups takes in resourceConfig and returns parsed groups.
+func ParseGroups(resourceConfig cliflag.ConfigurationMap) ([]string, error) {
+	groups := []string{}
+	for key := range resourceConfig {
+		if _, ok := groupVersionMatchers[key]; ok {
+			continue
+		}
+		tokens := strings.Split(key, "/")
+		if len(tokens) != 2 && len(tokens) != 3 {
+			return groups, fmt.Errorf("runtime-config invalid key %s", key)
+		}
+		groupVersionString := tokens[0] + "/" + tokens[1]
+		groupVersion, err := schema.ParseGroupVersion(groupVersionString)
+		if err != nil {
+			return nil, fmt.Errorf("runtime-config invalid key %s", key)
+		}
+		groups = append(groups, groupVersion.Group)
+	}
+
+	return groups, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/OWNERS

@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts

e2e/vendor/k8s.io/apiserver/pkg/server/routes/debugsocket.go

@@ -0,0 +1,82 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"os"
+	"path"
+)
+
+// DebugSocket installs profiling and debugflag as a Unix-Domain socket.
+type DebugSocket struct {
+	path string
+	mux  *http.ServeMux
+}
+
+// NewDebugSocket creates a new DebugSocket for the given path.
+func NewDebugSocket(path string) *DebugSocket {
+	return &DebugSocket{
+		path: path,
+		mux:  http.NewServeMux(),
+	}
+}
+
+// InstallProfiling installs profiling endpoints in the socket.
+func (s *DebugSocket) InstallProfiling() {
+	s.mux.HandleFunc("/debug/pprof", redirectTo("/debug/pprof/"))
+	s.mux.HandleFunc("/debug/pprof/", pprof.Index)
+	s.mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+	s.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+	s.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	s.mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+}
+
+// InstallDebugFlag installs debug flag endpoints in the socket.
+func (s *DebugSocket) InstallDebugFlag(flag string, handler func(http.ResponseWriter, *http.Request)) {
+	f := DebugFlags{}
+	s.mux.HandleFunc("/debug/flags", f.Index)
+	s.mux.HandleFunc("/debug/flags/", f.Index)
+
+	url := path.Join("/debug/flags", flag)
+	s.mux.HandleFunc(url, handler)
+
+	f.addFlag(flag)
+}
+
+// Run starts the server and waits for stopCh to be closed to close the server.
+func (s *DebugSocket) Run(stopCh <-chan struct{}) error {
+	if err := os.Remove(s.path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to remove (%v): %v", s.path, err)
+	}
+
+	l, err := net.Listen("unix", s.path)
+	if err != nil {
+		return fmt.Errorf("listen error (%v): %v", s.path, err)
+	}
+	defer l.Close()
+
+	srv := http.Server{Handler: s.mux}
+	go func() {
+		<-stopCh
+		srv.Close()
+	}()
+	return srv.Serve(l)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package routes holds a collection of optional genericapiserver http handlers.
+package routes // import "k8s.io/apiserver/pkg/server/routes"

e2e/vendor/k8s.io/apiserver/pkg/server/routes/flags.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"sync"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+var (
+	lock            = &sync.RWMutex{}
+	registeredFlags = map[string]debugFlag{}
+)
+
+// DebugFlags adds handlers for flags under /debug/flags.
+type DebugFlags struct {
+}
+
+// Install registers the APIServer's flags handler.
+func (f DebugFlags) Install(c *mux.PathRecorderMux, flag string, handler func(http.ResponseWriter, *http.Request)) {
+	c.UnlistedHandle("/debug/flags", http.HandlerFunc(f.Index))
+	c.UnlistedHandlePrefix("/debug/flags/", http.HandlerFunc(f.Index))
+
+	url := path.Join("/debug/flags", flag)
+	c.UnlistedHandleFunc(url, handler)
+
+	f.addFlag(flag)
+}
+
+// Index responds with the `/debug/flags` request.
+// For example, "/debug/flags/v" serves the "--v" flag.
+// Index responds to a request for "/debug/flags/" with an HTML page
+// listing the available flags.
+func (f DebugFlags) Index(w http.ResponseWriter, r *http.Request) {
+	lock.RLock()
+	defer lock.RUnlock()
+	if err := indexTmpl.Execute(w, registeredFlags); err != nil {
+		klog.Error(err)
+	}
+}
+
+var indexTmpl = template.Must(template.New("index").Parse(`<html>
+<head>
+<title>/debug/flags/</title>
+</head>
+<body>
+/debug/flags/<br>
+<br>
+flags:<br>
+<table>
+{{range .}}
+<tr>{{.Flag}}<br>
+{{end}}
+</table>
+<br>
+full flags configurable<br>
+</body>
+</html>
+`))
+
+type debugFlag struct {
+	Flag string
+}
+
+func (f DebugFlags) addFlag(flag string) {
+	lock.Lock()
+	defer lock.Unlock()
+	registeredFlags[flag] = debugFlag{flag}
+}
+
+// StringFlagSetterFunc is a func used for setting string type flag.
+type StringFlagSetterFunc func(string) (string, error)
+
+// StringFlagPutHandler wraps an http Handler to set string type flag.
+func StringFlagPutHandler(setter StringFlagSetterFunc) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch {
+		case req.Method == "PUT":
+			body, err := ioutil.ReadAll(req.Body)
+			if err != nil {
+				writePlainText(http.StatusBadRequest, "error reading request body: "+err.Error(), w)
+				return
+			}
+			defer req.Body.Close()
+			response, err := setter(string(body))
+			if err != nil {
+				writePlainText(http.StatusBadRequest, err.Error(), w)
+				return
+			}
+			writePlainText(http.StatusOK, response, w)
+			return
+		default:
+			writePlainText(http.StatusNotAcceptable, "unsupported http method", w)
+			return
+		}
+	})
+}
+
+// writePlainText renders a simple string response.
+func writePlainText(statusCode int, text string, w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(statusCode)
+	fmt.Fprintln(w, text)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/index.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// ListedPathProvider is an interface for providing paths that should be reported at /.
+type ListedPathProvider interface {
+	// ListedPaths is an alphabetically sorted list of paths to be reported at /.
+	ListedPaths() []string
+}
+
+// ListedPathProviders is a convenient way to combine multiple ListedPathProviders
+type ListedPathProviders []ListedPathProvider
+
+// ListedPaths unions and sorts the included paths.
+func (p ListedPathProviders) ListedPaths() []string {
+	ret := sets.String{}
+	for _, provider := range p {
+		for _, path := range provider.ListedPaths() {
+			ret.Insert(path)
+		}
+	}
+
+	return ret.List()
+}
+
+// Index provides a webservice for the http root / listing all known paths.
+type Index struct{}
+
+// Install adds the Index webservice to the given mux.
+func (i Index) Install(pathProvider ListedPathProvider, mux *mux.PathRecorderMux) {
+	handler := IndexLister{StatusCode: http.StatusOK, PathProvider: pathProvider}
+
+	mux.UnlistedHandle("/", handler)
+	mux.UnlistedHandle("/index.html", handler)
+}
+
+// IndexLister lists the available indexes with the status code provided
+type IndexLister struct {
+	StatusCode   int
+	PathProvider ListedPathProvider
+}
+
+// ServeHTTP serves the available paths.
+func (i IndexLister) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	responsewriters.WriteRawJSON(i.StatusCode, metav1.RootPaths{Paths: i.PathProvider.ListedPaths()}, w)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	handlersmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
+	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/mux"
+	cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics"
+	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
+	flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	peerproxymetrics "k8s.io/apiserver/pkg/util/peerproxy/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// DefaultMetrics installs the default prometheus metrics handler
+type DefaultMetrics struct{}
+
+// Install adds the DefaultMetrics handler
+func (m DefaultMetrics) Install(c *mux.PathRecorderMux) {
+	register()
+	c.Handle("/metrics", legacyregistry.Handler())
+}
+
+// MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method
+// which resets the metrics.
+type MetricsWithReset struct{}
+
+// Install adds the MetricsWithReset handler
+func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
+	register()
+	c.Handle("/metrics", legacyregistry.HandlerWithReset())
+}
+
+// register apiserver and etcd metrics
+func register() {
+	apimetrics.Register()
+	cachermetrics.Register()
+	etcd3metrics.Register()
+	flowcontrolmetrics.Register()
+	peerproxymetrics.Register()
+	handlersmetrics.Register()
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	restful "github.com/emicklei/go-restful/v3"
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/server/mux"
+	builder2 "k8s.io/kube-openapi/pkg/builder"
+	"k8s.io/kube-openapi/pkg/builder3"
+	"k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/common/restfuladapter"
+	"k8s.io/kube-openapi/pkg/handler"
+	"k8s.io/kube-openapi/pkg/handler3"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+// OpenAPI installs spec endpoints for each web service.
+type OpenAPI struct {
+	Config   *common.Config
+	V3Config *common.OpenAPIV3Config
+}
+
+// Install adds the SwaggerUI webservice to the given mux.
+func (oa OpenAPI) InstallV2(c *restful.Container, mux *mux.PathRecorderMux) (*handler.OpenAPIService, *spec.Swagger) {
+	spec, err := builder2.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(c.RegisteredWebServices()), oa.Config)
+	if err != nil {
+		klog.Fatalf("Failed to build open api spec for root: %v", err)
+	}
+	spec.Definitions = handler.PruneDefaults(spec.Definitions)
+	openAPIVersionedService := handler.NewOpenAPIService(spec)
+	openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
+
+	return openAPIVersionedService, spec
+}
+
+// InstallV3 adds the static group/versions defined in the RegisteredWebServices to the OpenAPI v3 spec
+func (oa OpenAPI) InstallV3(c *restful.Container, mux *mux.PathRecorderMux) *handler3.OpenAPIService {
+	openAPIVersionedService := handler3.NewOpenAPIService()
+	err := openAPIVersionedService.RegisterOpenAPIV3VersionedService("/openapi/v3", mux)
+	if err != nil {
+		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
+	}
+
+	grouped := make(map[string][]*restful.WebService)
+
+	for _, t := range c.RegisteredWebServices() {
+		// Strip the "/" prefix from the name
+		gvName := t.RootPath()[1:]
+		grouped[gvName] = []*restful.WebService{t}
+	}
+
+	for gv, ws := range grouped {
+		spec, err := builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(ws), oa.V3Config)
+		if err != nil {
+			klog.Errorf("Failed to build OpenAPI v3 for group %s, %q", gv, err)
+
+		}
+		openAPIVersionedService.UpdateGroupVersion(gv, spec)
+	}
+	return openAPIVersionedService
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/profiling.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+	"net/http/pprof"
+
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// Profiling adds handlers for pprof under /debug/pprof.
+type Profiling struct{}
+
+// Install adds the Profiling webservice to the given mux.
+func (d Profiling) Install(c *mux.PathRecorderMux) {
+	c.UnlistedHandleFunc("/debug/pprof", redirectTo("/debug/pprof/"))
+	c.UnlistedHandlePrefix("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	c.UnlistedHandleFunc("/debug/pprof/profile", pprof.Profile)
+	c.UnlistedHandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	c.UnlistedHandleFunc("/debug/pprof/trace", pprof.Trace)
+}
+
+// redirectTo redirects request to a certain destination.
+func redirectTo(to string) func(http.ResponseWriter, *http.Request) {
+	return func(rw http.ResponseWriter, req *http.Request) {
+		http.Redirect(rw, req, to, http.StatusFound)
+	}
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routes/version.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful/v3"
+
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+// Version provides a webservice with version information.
+type Version struct {
+	Version *version.Info
+}
+
+// Install registers the APIServer's `/version` handler.
+func (v Version) Install(c *restful.Container) {
+	if v.Version == nil {
+		return
+	}
+
+	// Set up a service to return the git code version.
+	versionWS := new(restful.WebService)
+	versionWS.Path("/version")
+	versionWS.Doc("git code version from which this is built")
+	versionWS.Route(
+		versionWS.GET("/").To(v.handleVersion).
+			Doc("get the code version").
+			Operation("getCodeVersion").
+			Produces(restful.MIME_JSON).
+			Consumes(restful.MIME_JSON).
+			Writes(version.Info{}))
+
+	c.Add(versionWS)
+}
+
+// handleVersion writes the server's version information.
+func (v Version) handleVersion(req *restful.Request, resp *restful.Response) {
+	responsewriters.WriteRawJSON(http.StatusOK, *v.Version, resp.ResponseWriter)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/routine/routine.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routine
+
+import (
+	"context"
+	"net/http"
+
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+type taskKeyType int
+
+const taskKey taskKeyType = iota
+
+type Task struct {
+	Func func()
+}
+
+func WithTask(parent context.Context, t *Task) context.Context {
+	return request.WithValue(parent, taskKey, t)
+}
+
+// AppendTask appends a task executed after completion of existing task.
+// It is a no-op if there is no existing task.
+func AppendTask(ctx context.Context, t *Task) bool {
+	if existTask := TaskFrom(ctx); existTask != nil && existTask.Func != nil {
+		existFunc := existTask.Func
+		existTask.Func = func() {
+			existFunc()
+			t.Func()
+		}
+		return true
+	}
+	return false
+}
+
+func TaskFrom(ctx context.Context) *Task {
+	t, _ := ctx.Value(taskKey).(*Task)
+	return t
+}
+
+// WithRoutine returns an http.Handler that executes preparation of long running requests (i.e. watches)
+// in a separate Goroutine and then serves the long running request in the main Goroutine. Doing so allows
+// freeing stack memory used in preparation Goroutine for better memory efficiency.
+func WithRoutine(handler http.Handler, longRunning request.LongRunningRequestCheck) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		requestInfo, _ := request.RequestInfoFrom(ctx)
+		if !longRunning(req, requestInfo) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		req = req.WithContext(WithTask(ctx, &Task{}))
+		panicCh := make(chan any, 1)
+		go func() {
+			defer func() {
+				if r := recover(); r != nil {
+					panicCh <- r
+				}
+				close(panicCh)
+			}()
+			handler.ServeHTTP(w, req)
+		}()
+
+		if p, ok := <-panicCh; ok {
+			panic(p)
+		}
+
+		ctx = req.Context()
+		if t := TaskFrom(ctx); t != nil && t.Func != nil {
+			t.Func()
+		}
+
+	})
+}

e2e/vendor/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -0,0 +1,306 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/net/http2"
+	"k8s.io/component-base/cli/flag"
+	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+)
+
+const (
+	defaultKeepAlivePeriod = 3 * time.Minute
+)
+
+// tlsConfig produces the tls.Config to serve with.
+func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, error) {
+	tlsConfig := &tls.Config{
+		// Can't use SSLv3 because of POODLE and BEAST
+		// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+		// Can't use TLSv1.1 because of RC4 cipher usage
+		MinVersion: tls.VersionTLS12,
+		// enable HTTP2 for go's 1.7 HTTP Server
+		NextProtos: []string{"h2", "http/1.1"},
+	}
+
+	// these are static aspects of the tls.Config
+	if s.DisableHTTP2 {
+		klog.Info("Forcing use of http/1.1 only")
+		tlsConfig.NextProtos = []string{"http/1.1"}
+	}
+	if s.MinTLSVersion > 0 {
+		tlsConfig.MinVersion = s.MinTLSVersion
+	}
+	if len(s.CipherSuites) > 0 {
+		tlsConfig.CipherSuites = s.CipherSuites
+		insecureCiphers := flag.InsecureTLSCiphers()
+		for i := 0; i < len(s.CipherSuites); i++ {
+			for cipherName, cipherID := range insecureCiphers {
+				if s.CipherSuites[i] == cipherID {
+					klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
+				}
+			}
+		}
+	}
+
+	if s.ClientCA != nil {
+		// Populate PeerCertificates in requests, but don't reject connections without certificates
+		// This allows certificates to be validated by authenticators, while still allowing other auth types
+		tlsConfig.ClientAuth = tls.RequestClientCert
+	}
+
+	if s.ClientCA != nil || s.Cert != nil || len(s.SNICerts) > 0 {
+		dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
+			tlsConfig,
+			s.ClientCA,
+			s.Cert,
+			s.SNICerts,
+			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
+		)
+
+		if s.ClientCA != nil {
+			s.ClientCA.AddListener(dynamicCertificateController)
+		}
+		if s.Cert != nil {
+			s.Cert.AddListener(dynamicCertificateController)
+		}
+		// generate a context from stopCh. This is to avoid modifying files which are relying on apiserver
+		// TODO: See if we can pass ctx to the current method
+		ctx, cancel := context.WithCancel(context.Background())
+		go func() {
+			select {
+			case <-stopCh:
+				cancel() // stopCh closed, so cancel our context
+			case <-ctx.Done():
+			}
+		}()
+		// start controllers if possible
+		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(ctx); err != nil {
+				klog.Warningf("Initial population of client CA failed: %v", err)
+			}
+
+			go controller.Run(ctx, 1)
+		}
+		if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(ctx); err != nil {
+				klog.Warningf("Initial population of default serving certificate failed: %v", err)
+			}
+
+			go controller.Run(ctx, 1)
+		}
+		for _, sniCert := range s.SNICerts {
+			sniCert.AddListener(dynamicCertificateController)
+			if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok {
+				// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+				// Files are required to be populated already, so this is for convenience.
+				if err := controller.RunOnce(ctx); err != nil {
+					klog.Warningf("Initial population of SNI serving certificate failed: %v", err)
+				}
+
+				go controller.Run(ctx, 1)
+			}
+		}
+
+		// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+		// Files are required to be populated already, so this is for convenience.
+		if err := dynamicCertificateController.RunOnce(); err != nil {
+			klog.Warningf("Initial population of dynamic certificates failed: %v", err)
+		}
+		go dynamicCertificateController.Run(1, stopCh)
+
+		tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
+	}
+
+	return tlsConfig, nil
+}
+
+// Serve runs the secure http server. It fails only if certificates cannot be loaded or the initial listen call fails.
+// The actual server loop (stoppable by closing stopCh) runs in a go routine, i.e. Serve does not block.
+// It returns a stoppedCh that is closed when all non-hijacked active requests have been processed.
+// It returns a listenerStoppedCh that is closed when the underlying http Server has stopped listening.
+func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) (<-chan struct{}, <-chan struct{}, error) {
+	if s.Listener == nil {
+		return nil, nil, fmt.Errorf("listener must not be nil")
+	}
+
+	tlsConfig, err := s.tlsConfig(stopCh)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+		TLSConfig:      tlsConfig,
+
+		IdleTimeout:       90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+		ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound
+	}
+
+	// At least 99% of serialized resources in surveyed clusters were smaller than 256kb.
+	// This should be big enough to accommodate most API POST requests in a single frame,
+	// and small enough to allow a per connection buffer of this size multiplied by `MaxConcurrentStreams`.
+	const resourceBody99Percentile = 256 * 1024
+
+	http2Options := &http2.Server{
+		IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+	}
+
+	// shrink the per-stream buffer and max framesize from the 1MB default while still accommodating most API POST requests in a single frame
+	http2Options.MaxUploadBufferPerStream = resourceBody99Percentile
+	http2Options.MaxReadFrameSize = resourceBody99Percentile
+
+	// use the overridden concurrent streams setting or make the default of 250 explicit so we can size MaxUploadBufferPerConnection appropriately
+	if s.HTTP2MaxStreamsPerConnection > 0 {
+		http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection)
+	} else {
+		// match http2.initialMaxConcurrentStreams used by clients
+		// this makes it so that a malicious client can only open 400 streams before we forcibly close the connection
+		// https://github.com/golang/net/commit/b225e7ca6dde1ef5a5ae5ce922861bda011cfabd
+		http2Options.MaxConcurrentStreams = 100
+	}
+
+	// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
+	http2Options.MaxUploadBufferPerConnection = http2Options.MaxUploadBufferPerStream * int32(http2Options.MaxConcurrentStreams)
+
+	if !s.DisableHTTP2 {
+		// apply settings to the server
+		if err := http2.ConfigureServer(secureServer, http2Options); err != nil {
+			return nil, nil, fmt.Errorf("error configuring http2: %v", err)
+		}
+	}
+
+	// use tlsHandshakeErrorWriter to handle messages of tls handshake error
+	tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
+	tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
+	secureServer.ErrorLog = tlsErrorLogger
+
+	klog.Infof("Serving securely on %s", secureServer.Addr)
+	return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
+}
+
+// RunServer spawns a go-routine continuously serving until the stopCh is
+// closed.
+// It returns a stoppedCh that is closed when all non-hijacked active requests
+// have been processed.
+// This function does not block
+// TODO: make private when insecure serving is gone from the kube-apiserver
+func RunServer(
+	server *http.Server,
+	ln net.Listener,
+	shutDownTimeout time.Duration,
+	stopCh <-chan struct{},
+) (<-chan struct{}, <-chan struct{}, error) {
+	if ln == nil {
+		return nil, nil, fmt.Errorf("listener must not be nil")
+	}
+
+	// Shutdown server gracefully.
+	serverShutdownCh, listenerStoppedCh := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(serverShutdownCh)
+		<-stopCh
+		ctx, cancel := context.WithTimeout(context.Background(), shutDownTimeout)
+		server.Shutdown(ctx)
+		cancel()
+	}()
+
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer close(listenerStoppedCh)
+
+		var listener net.Listener
+		listener = tcpKeepAliveListener{ln}
+		if server.TLSConfig != nil {
+			listener = tls.NewListener(listener, server.TLSConfig)
+		}
+
+		err := server.Serve(listener)
+
+		msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String())
+		select {
+		case <-stopCh:
+			klog.Info(msg)
+		default:
+			panic(fmt.Sprintf("%s due to error: %v", msg, err))
+		}
+	}()
+
+	return serverShutdownCh, listenerStoppedCh, nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+//
+// Copied from Go 1.7.2 net/http/server.go
+type tcpKeepAliveListener struct {
+	net.Listener
+}
+
+func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
+	c, err := ln.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	if tc, ok := c.(*net.TCPConn); ok {
+		tc.SetKeepAlive(true)
+		tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
+	}
+	return c, nil
+}
+
+// tlsHandshakeErrorWriter writes TLS handshake errors to klog with
+// trace level - V(5), to avoid flooding of tls handshake errors.
+type tlsHandshakeErrorWriter struct {
+	out io.Writer
+}
+
+const tlsHandshakeErrorPrefix = "http: TLS handshake error"
+
+func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
+	if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
+		klog.V(5).Info(string(p))
+		metrics.TLSHandshakeErrors.Inc()
+		return len(p), nil
+	}
+
+	// for non tls handshake error, log it as usual
+	return w.out.Write(p)
+}

e2e/vendor/k8s.io/apiserver/pkg/server/signal.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+	"os/signal"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+var shutdownHandler chan os.Signal
+
+// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
+// which is closed on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
+func SetupSignalHandler() <-chan struct{} {
+	return SetupSignalContext().Done()
+}
+
+// SetupSignalContext is same as SetupSignalHandler, but a context.Context is returned.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
+func SetupSignalContext() context.Context {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	shutdownHandler = make(chan os.Signal, 2)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signal.Notify(shutdownHandler, shutdownSignals...)
+	go func() {
+		<-shutdownHandler
+		cancel()
+		<-shutdownHandler
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx
+}
+
+// RequestShutdown emulates a received event that is considered as shutdown signal (SIGTERM/SIGINT)
+// This returns whether a handler was notified
+func RequestShutdown() bool {
+	if shutdownHandler != nil {
+		select {
+		case shutdownHandler <- shutdownSignals[0]:
+			return true
+		default:
+		}
+	}
+
+	return false
+}

e2e/vendor/k8s.io/apiserver/pkg/server/signal_posix.go

@@ -0,0 +1,27 @@
+//go:build !windows
+// +build !windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"syscall"
+)
+
+var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}

e2e/vendor/k8s.io/apiserver/pkg/server/signal_windows.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+)
+
+var shutdownSignals = []os.Signal{os.Interrupt}

e2e/vendor/k8s.io/apiserver/pkg/server/storage/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package storage contains the plumbing to setup the etcd storage of the apiserver.
+package storage // import "k8s.io/apiserver/pkg/server/storage"

e2e/vendor/k8s.io/apiserver/pkg/server/storage/resource_config.go

@@ -0,0 +1,153 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// APIResourceConfigSource is the interface to determine which groups and versions are enabled
+type APIResourceConfigSource interface {
+	ResourceEnabled(resource schema.GroupVersionResource) bool
+	AnyResourceForGroupEnabled(group string) bool
+}
+
+var _ APIResourceConfigSource = &ResourceConfig{}
+
+type ResourceConfig struct {
+	GroupVersionConfigs map[schema.GroupVersion]bool
+	ResourceConfigs     map[schema.GroupVersionResource]bool
+}
+
+func NewResourceConfig() *ResourceConfig {
+	return &ResourceConfig{GroupVersionConfigs: map[schema.GroupVersion]bool{}, ResourceConfigs: map[schema.GroupVersionResource]bool{}}
+}
+
+// DisableMatchingVersions disables all group/versions for which the matcher function returns true.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) DisableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for version := range o.GroupVersionConfigs {
+		if matcher(version) {
+			o.GroupVersionConfigs[version] = false
+			o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+		}
+	}
+}
+
+// EnableMatchingVersions enables all group/versions for which the matcher function returns true.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) EnableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for version := range o.GroupVersionConfigs {
+		if matcher(version) {
+			o.GroupVersionConfigs[version] = true
+			o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+		}
+	}
+}
+
+// resourceMatcherForVersion matches resources in the specified version
+func resourceMatcherForVersion(gv schema.GroupVersion) func(gvr schema.GroupVersionResource) bool {
+	return func(gvr schema.GroupVersionResource) bool {
+		return gv == gvr.GroupVersion()
+	}
+}
+
+// removeMatchingResourcePreferences removes individual resource preferences that match.  This is useful when an override of a version or level enablement should
+// override the previously individual preferences.
+func (o *ResourceConfig) removeMatchingResourcePreferences(matcher func(gvr schema.GroupVersionResource) bool) {
+	keysToRemove := []schema.GroupVersionResource{}
+	for k := range o.ResourceConfigs {
+		if matcher(k) {
+			keysToRemove = append(keysToRemove, k)
+		}
+	}
+	for _, k := range keysToRemove {
+		delete(o.ResourceConfigs, k)
+	}
+}
+
+// DisableVersions disables the versions entirely.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) DisableVersions(versions ...schema.GroupVersion) {
+	for _, version := range versions {
+		o.GroupVersionConfigs[version] = false
+
+		// a preference about a version takes priority over the previously set resources
+		o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+	}
+}
+
+// EnableVersions enables all resources in a given groupVersion.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) EnableVersions(versions ...schema.GroupVersion) {
+	for _, version := range versions {
+		o.GroupVersionConfigs[version] = true
+
+		// a preference about a version takes priority over the previously set resources
+		o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+	}
+
+}
+
+// TODO this must be removed and we enable/disable individual resources.
+func (o *ResourceConfig) versionEnabled(version schema.GroupVersion) bool {
+	enabled, _ := o.GroupVersionConfigs[version]
+	return enabled
+}
+
+func (o *ResourceConfig) DisableResources(resources ...schema.GroupVersionResource) {
+	for _, resource := range resources {
+		o.ResourceConfigs[resource] = false
+	}
+}
+
+func (o *ResourceConfig) EnableResources(resources ...schema.GroupVersionResource) {
+	for _, resource := range resources {
+		o.ResourceConfigs[resource] = true
+	}
+}
+
+func (o *ResourceConfig) ResourceEnabled(resource schema.GroupVersionResource) bool {
+	// if a resource is explicitly set, that takes priority over the preference of the version.
+	resourceEnabled, explicitlySet := o.ResourceConfigs[resource]
+	if explicitlySet {
+		return resourceEnabled
+	}
+
+	if !o.versionEnabled(resource.GroupVersion()) {
+		return false
+	}
+	// they are enabled by default.
+	return true
+}
+
+func (o *ResourceConfig) AnyResourceForGroupEnabled(group string) bool {
+	for version := range o.GroupVersionConfigs {
+		if version.Group == group {
+			if o.versionEnabled(version) {
+				return true
+			}
+		}
+	}
+	for resource := range o.ResourceConfigs {
+		if resource.Group == group && o.ResourceEnabled(resource) {
+			return true
+		}
+	}
+
+	return false
+}

e2e/vendor/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go

@@ -0,0 +1,212 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
+	version "k8s.io/component-base/version"
+)
+
+type ResourceEncodingConfig interface {
+	// StorageEncoding returns the serialization format for the resource.
+	// TODO this should actually return a GroupVersionKind since you can logically have multiple "matching" Kinds
+	// For now, it returns just the GroupVersion for consistency with old behavior
+	StorageEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
+
+	// InMemoryEncodingFor returns the groupVersion for the in memory representation the storage should convert to.
+	InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
+}
+
+type CompatibilityResourceEncodingConfig interface {
+	BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
+}
+
+type DefaultResourceEncodingConfig struct {
+	// resources records the overriding encoding configs for individual resources.
+	resources        map[schema.GroupResource]*OverridingResourceEncoding
+	scheme           *runtime.Scheme
+	effectiveVersion version.EffectiveVersion
+}
+
+type OverridingResourceEncoding struct {
+	ExternalResourceEncoding schema.GroupVersion
+	InternalResourceEncoding schema.GroupVersion
+}
+
+var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
+
+func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
+	return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
+}
+
+func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
+	o.resources[resourceBeingStored] = &OverridingResourceEncoding{
+		ExternalResourceEncoding: externalEncodingVersion,
+		InternalResourceEncoding: internalVersion,
+	}
+}
+
+func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
+	o.effectiveVersion = effectiveVersion
+}
+
+func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.ExternalResourceEncoding, nil
+	}
+
+	// return the most preferred external version for the group
+	return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
+}
+
+func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	// Always respect overrides
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.ExternalResourceEncoding, nil
+	}
+
+	return emulatedStorageVersion(
+		o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
+		example,
+		o.effectiveVersion,
+		o.scheme)
+}
+
+func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.InternalResourceEncoding, nil
+	}
+	return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
+}
+
+// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
+type introducedInterface interface {
+	APILifecycleIntroduced() (major, minor int)
+}
+
+type replacementInterface interface {
+	APILifecycleReplacement() schema.GroupVersionKind
+}
+
+func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
+	if example == nil || effectiveVersion == nil {
+		return binaryVersionOfResource, nil
+	}
+
+	// Look up example in scheme to find all objects of the same Group-Kind
+	// Use the highest priority version for that group-kind whose lifecycle window
+	// includes the current emulation version.
+	// If no version is found, use the binary version
+	// (in this case the API should be disabled anyway)
+	gvks, _, err := scheme.ObjectKinds(example)
+	if err != nil {
+		return schema.GroupVersion{}, err
+	}
+
+	var gvk schema.GroupVersionKind
+	for _, item := range gvks {
+		if item.Group != binaryVersionOfResource.Group {
+			continue
+		}
+
+		gvk = item
+		break
+	}
+
+	if len(gvk.Kind) == 0 {
+		return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
+	}
+
+	// VersionsForGroupKind returns versions in priority order
+	versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind})
+
+	compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
+
+	for _, gv := range versions {
+		if gv.Version == runtime.APIVersionInternal {
+			continue
+		}
+
+		gvk := schema.GroupVersionKind{
+			Group:   gv.Group,
+			Version: gv.Version,
+			Kind:    gvk.Kind,
+		}
+
+		exampleOfGVK, err := scheme.New(gvk)
+		if err != nil {
+			return schema.GroupVersion{}, err
+		}
+
+		// If it was introduced after current compatibility version, don't use it
+		// skip the introduced check for test when currentVersion is 0.0 to test all apis
+		if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
+
+			// Skip versions that have a replacement.
+			// This can be used to override this storage version selection by
+			// marking a storage version has having a replacement and preventing a
+			// that storage version from being selected.
+			if _, hasReplacement := exampleOfGVK.(replacementInterface); hasReplacement {
+				continue
+			}
+			// API resource lifecycles should be relative to k8s api version
+			majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+			introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+			if introducedVer.GreaterThan(compatibilityVersion) {
+				continue
+			}
+		}
+
+		// versions is returned in priority order, so just use first result
+		return gvk.GroupVersion(), nil
+	}
+
+	// Getting here means we're serving a version that is unknown to the
+	// min-compatibility-version server.
+	//
+	// This is only expected to happen when serving an alpha API type due
+	// to missing pre-release lifecycle information
+	// (which doesn't happen by default), or when emulation-version and
+	// min-compatibility-version are several versions apart so a beta or GA API
+	// was being served which didn't exist at all in min-compatibility-version.
+	//
+	// In the alpha case - we do not support compatibility versioning of
+	// alpha types and recommend users do not mix the two.
+	// In the skip-level case - The version of apiserver we are retaining
+	// compatibility with has no knowledge of the type,
+	// so storing it in another type is no issue.
+	return binaryVersionOfResource, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/storage/storage_codec.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"mime"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+// StorageCodecConfig are the arguments passed to newStorageCodecFn
+type StorageCodecConfig struct {
+	StorageMediaType  string
+	StorageSerializer runtime.StorageSerializer
+	StorageVersion    schema.GroupVersion
+	MemoryVersion     schema.GroupVersion
+	Config            storagebackend.Config
+
+	EncoderDecoratorFn func(runtime.Encoder) runtime.Encoder
+	DecoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
+}
+
+// NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested
+// storage and memory versions.
+func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) {
+	mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
+	}
+
+	supportedMediaTypes := opts.StorageSerializer.SupportedMediaTypes()
+	serializer, ok := runtime.SerializerInfoForMediaType(supportedMediaTypes, mediaType)
+	if !ok {
+		supportedMediaTypeList := make([]string, len(supportedMediaTypes))
+		for i, mediaType := range supportedMediaTypes {
+			supportedMediaTypeList[i] = mediaType.MediaType
+		}
+		return nil, nil, fmt.Errorf("unable to find serializer for %q, supported media types: %v", mediaType, supportedMediaTypeList)
+	}
+
+	s := serializer.Serializer
+
+	// Give callers the opportunity to wrap encoders and decoders.  For decoders, each returned decoder will
+	// be passed to the recognizer so that multiple decoders are available.
+	var encoder runtime.Encoder = s
+	if opts.EncoderDecoratorFn != nil {
+		encoder = opts.EncoderDecoratorFn(encoder)
+	}
+	decoders := []runtime.Decoder{
+		// selected decoder as the primary
+		s,
+		// universal deserializer as a fallback
+		opts.StorageSerializer.UniversalDeserializer(),
+		// base64-wrapped universal deserializer as a last resort.
+		// this allows reading base64-encoded protobuf, which should only exist if etcd2+protobuf was used at some point.
+		// data written that way could exist in etcd2, or could have been migrated to etcd3.
+		// TODO: flag this type of data if we encounter it, require migration (read to decode, write to persist using a supported encoder), and remove in 1.8
+		runtime.NewBase64Serializer(nil, opts.StorageSerializer.UniversalDeserializer()),
+	}
+	if opts.DecoderDecoratorFn != nil {
+		decoders = opts.DecoderDecoratorFn(decoders)
+	}
+
+	encodeVersioner := runtime.NewMultiGroupVersioner(
+		opts.StorageVersion,
+		schema.GroupKind{Group: opts.StorageVersion.Group},
+		schema.GroupKind{Group: opts.MemoryVersion.Group},
+	)
+
+	// Ensure the storage receives the correct version.
+	encoder = opts.StorageSerializer.EncoderForVersion(
+		encoder,
+		encodeVersioner,
+	)
+	decoder := opts.StorageSerializer.DecoderToVersion(
+		recognizer.NewDecoder(decoders...),
+		runtime.NewCoercingMultiGroupVersioner(
+			opts.MemoryVersion,
+			schema.GroupKind{Group: opts.MemoryVersion.Group},
+			schema.GroupKind{Group: opts.StorageVersion.Group},
+		),
+	)
+
+	return runtime.NewCodec(encoder, decoder), encodeVersioner, nil
+}

e2e/vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -0,0 +1,372 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"os"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/klog/v2"
+)
+
+// Backend describes the storage servers, the information here should be enough
+// for health validations.
+type Backend struct {
+	// the url of storage backend like: https://etcd.domain:2379
+	Server string
+	// the required tls config
+	TLSConfig *tls.Config
+}
+
+// StorageFactory is the interface to locate the storage for a given GroupResource
+type StorageFactory interface {
+	// New finds the storage destination for the given group and resource. It will
+	// return an error if the group has no storage destination configured.
+	NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
+
+	// ResourcePrefix returns the overridden resource prefix for the GroupResource
+	// This allows for cohabitation of resources with different native types and provides
+	// centralized control over the shape of etcd directories
+	ResourcePrefix(groupResource schema.GroupResource) string
+
+	// Configs gets configurations for all of registered storage destinations.
+	Configs() []storagebackend.Config
+
+	// Backends gets all backends for all registered storage destinations.
+	// Used for getting all instances for health validations.
+	// Deprecated: Use Configs instead
+	Backends() []Backend
+}
+
+// DefaultStorageFactory takes a GroupResource and returns back its storage interface.  This result includes:
+// 1. Merged etcd config, including: auth, server locations, prefixes
+// 2. Resource encodings for storage: group,version,kind to store as
+// 3. Cohabitating default: some resources like hpa are exposed through multiple APIs.  They must agree on 1 and 2
+type DefaultStorageFactory struct {
+	// StorageConfig describes how to create a storage backend in general.
+	// Its authentication information will be used for every storage.Interface returned.
+	StorageConfig storagebackend.Config
+
+	Overrides map[schema.GroupResource]groupResourceOverrides
+
+	DefaultResourcePrefixes map[schema.GroupResource]string
+
+	// DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used.
+	DefaultMediaType string
+
+	// DefaultSerializer is used to create encoders and decoders for the storage.Interface.
+	DefaultSerializer runtime.StorageSerializer
+
+	// ResourceEncodingConfig describes how to encode a particular GroupVersionResource
+	ResourceEncodingConfig ResourceEncodingConfig
+
+	// APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API
+	// This is discrete from resource enablement because those are separate concerns.  How this source is configured
+	// is left to the caller.
+	APIResourceConfigSource APIResourceConfigSource
+
+	// newStorageCodecFn exists to be overwritten for unit testing.
+	newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
+}
+
+type groupResourceOverrides struct {
+	// etcdLocation contains the list of "special" locations that are used for particular GroupResources
+	// These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource
+	etcdLocation []string
+	// etcdPrefix is the base location for a GroupResource.
+	etcdPrefix string
+	// etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location
+	// If empty, the default mapping is used.  If the default mapping doesn't contain an entry, it will use
+	// the ToLowered name of the resource, not including the group.
+	etcdResourcePrefix string
+	// mediaType is the desired serializer to choose. If empty, the default is chosen.
+	mediaType string
+	// serializer contains the list of "special" serializers for a GroupResource.  Resource=* means for the entire group
+	serializer runtime.StorageSerializer
+	// cohabitatingResources keeps track of which resources must be stored together.  This happens when we have multiple ways
+	// of exposing one set of concepts.  autoscaling.HPA and extensions.HPA as a for instance
+	// The order of the slice matters!  It is the priority order of lookup for finding a storage location
+	cohabitatingResources []schema.GroupResource
+	// encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized.
+	encoderDecoratorFn func(runtime.Encoder) runtime.Encoder
+	// decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of
+	// returned decoders will be priority for attempt to decode.
+	decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
+}
+
+// Apply overrides the provided config and options if the override has a value in that position
+func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) {
+	if len(o.etcdLocation) > 0 {
+		config.Transport.ServerList = o.etcdLocation
+	}
+	if len(o.etcdPrefix) > 0 {
+		config.Prefix = o.etcdPrefix
+	}
+
+	if len(o.mediaType) > 0 {
+		options.StorageMediaType = o.mediaType
+	}
+	if o.serializer != nil {
+		options.StorageSerializer = o.serializer
+	}
+	if o.encoderDecoratorFn != nil {
+		options.EncoderDecoratorFn = o.encoderDecoratorFn
+	}
+	if o.decoderDecoratorFn != nil {
+		options.DecoderDecoratorFn = o.decoderDecoratorFn
+	}
+}
+
+var _ StorageFactory = &DefaultStorageFactory{}
+
+const AllResources = "*"
+
+func NewDefaultStorageFactory(
+	config storagebackend.Config,
+	defaultMediaType string,
+	defaultSerializer runtime.StorageSerializer,
+	resourceEncodingConfig ResourceEncodingConfig,
+	resourceConfig APIResourceConfigSource,
+	specialDefaultResourcePrefixes map[schema.GroupResource]string,
+) *DefaultStorageFactory {
+	if len(defaultMediaType) == 0 {
+		defaultMediaType = runtime.ContentTypeJSON
+	}
+	return &DefaultStorageFactory{
+		StorageConfig:           config,
+		Overrides:               map[schema.GroupResource]groupResourceOverrides{},
+		DefaultMediaType:        defaultMediaType,
+		DefaultSerializer:       defaultSerializer,
+		ResourceEncodingConfig:  resourceEncodingConfig,
+		APIResourceConfigSource: resourceConfig,
+		DefaultResourcePrefixes: specialDefaultResourcePrefixes,
+
+		newStorageCodecFn: NewStorageCodec,
+	}
+}
+
+func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdLocation = location
+	s.Overrides[groupResource] = overrides
+}
+
+func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdPrefix = prefix
+	s.Overrides[groupResource] = overrides
+}
+
+// SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir.  You'll end up in `etcdPrefix/resourceEtcdPrefix`.
+func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdResourcePrefix = prefix
+	s.Overrides[groupResource] = overrides
+}
+
+func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) {
+	overrides := s.Overrides[groupResource]
+	overrides.mediaType = mediaType
+	overrides.serializer = serializer
+	s.Overrides[groupResource] = overrides
+}
+
+// AddCohabitatingResources links resources together the order of the slice matters!  its the priority order of lookup for finding a storage location
+func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) {
+	for _, groupResource := range groupResources {
+		overrides := s.Overrides[groupResource]
+		overrides.cohabitatingResources = groupResources
+		s.Overrides[groupResource] = overrides
+	}
+}
+
+func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) {
+	for _, groupResource := range groupResources {
+		overrides := s.Overrides[groupResource]
+		overrides.encoderDecoratorFn = encoderDecoratorFn
+		overrides.decoderDecoratorFn = decoderDecoratorFn
+		s.Overrides[groupResource] = overrides
+	}
+}
+
+func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource {
+	return schema.GroupResource{Group: resource.Group, Resource: AllResources}
+}
+
+func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource {
+	for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources {
+		// TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters
+		if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) {
+			return potentialStorageResource
+		}
+	}
+
+	return groupResource
+}
+
+// New finds the storage destination for the given group and resource. It will
+// return an error if the group has no storage destination configured.
+func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
+	chosenStorageResource := s.getStorageGroupResource(groupResource)
+
+	// operate on copy
+	storageConfig := s.StorageConfig
+	codecConfig := StorageCodecConfig{
+		StorageMediaType:  s.DefaultMediaType,
+		StorageSerializer: s.DefaultSerializer,
+	}
+
+	if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok {
+		override.Apply(&storageConfig, &codecConfig)
+	}
+	if override, ok := s.Overrides[chosenStorageResource]; ok {
+		override.Apply(&storageConfig, &codecConfig)
+	}
+
+	var err error
+	if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
+		codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(chosenStorageResource, example)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
+	if err != nil {
+		return nil, err
+	}
+
+	codecConfig.Config = storageConfig
+
+	storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
+	if err != nil {
+		return nil, err
+	}
+	klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config)
+
+	return storageConfig.ForResource(groupResource), nil
+}
+
+// Configs implements StorageFactory.
+func (s *DefaultStorageFactory) Configs() []storagebackend.Config {
+	return configs(s.StorageConfig, s.Overrides)
+}
+
+// Configs gets configurations for all of registered storage destinations.
+func Configs(storageConfig storagebackend.Config) []storagebackend.Config {
+	return configs(storageConfig, nil)
+}
+
+// Returns all storage configurations including those for group resource overrides
+func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config {
+	configs := []storagebackend.Config{storageConfig}
+
+	for _, override := range grOverrides {
+		if len(override.etcdLocation) == 0 {
+			continue
+		}
+		// copy
+		newConfig := storageConfig
+		override.Apply(&newConfig, &StorageCodecConfig{})
+		newConfig.Transport.ServerList = override.etcdLocation
+		configs = append(configs, newConfig)
+	}
+	return configs
+}
+
+// Backends implements StorageFactory.
+func (s *DefaultStorageFactory) Backends() []Backend {
+	return backends(s.StorageConfig, s.Overrides)
+}
+
+// Backends returns all backends for all registered storage destinations.
+// Used for getting all instances for health validations.
+// Deprecated: Validate health by passing storagebackend.Config directly to storagefactory.CreateProber.
+func Backends(storageConfig storagebackend.Config) []Backend {
+	return backends(storageConfig, nil)
+}
+
+func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend {
+	servers := sets.NewString(storageConfig.Transport.ServerList...)
+
+	for _, overrides := range grOverrides {
+		servers.Insert(overrides.etcdLocation...)
+	}
+
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 {
+		cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile)
+		if err != nil {
+			klog.Errorf("failed to load key pair while getting backends: %s", err)
+		} else {
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+	}
+	if len(storageConfig.Transport.TrustedCAFile) > 0 {
+		if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
+			klog.Errorf("failed to read ca file while getting backends: %s", err)
+		} else {
+			caPool := x509.NewCertPool()
+			caPool.AppendCertsFromPEM(caCert)
+			tlsConfig.RootCAs = caPool
+			tlsConfig.InsecureSkipVerify = false
+		}
+	}
+
+	backends := []Backend{}
+	for server := range servers {
+		backends = append(backends, Backend{
+			Server: server,
+			// We can't share TLSConfig across different backends to avoid races.
+			// For more details see: https://pr.k8s.io/59338
+			TLSConfig: tlsConfig.Clone(),
+		})
+	}
+	return backends
+}
+
+func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string {
+	chosenStorageResource := s.getStorageGroupResource(groupResource)
+	groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)]
+	exactResourceOverride := s.Overrides[chosenStorageResource]
+
+	etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource]
+	if len(groupOverride.etcdResourcePrefix) > 0 {
+		etcdResourcePrefix = groupOverride.etcdResourcePrefix
+	}
+	if len(exactResourceOverride.etcdResourcePrefix) > 0 {
+		etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix
+	}
+	if len(etcdResourcePrefix) == 0 {
+		etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource)
+	}
+
+	return etcdResourcePrefix
+}

e2e/vendor/k8s.io/apiserver/pkg/server/storage_readiness_hook.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/klog/v2"
+)
+
+// StorageReadinessHook implements PostStartHook functionality for checking readiness
+// of underlying storage for registered resources.
+type StorageReadinessHook struct {
+	timeout time.Duration
+
+	lock   sync.Mutex
+	checks map[string]func() error
+}
+
+// NewStorageReadinessHook created new StorageReadinessHook.
+func NewStorageReadinessHook(timeout time.Duration) *StorageReadinessHook {
+	return &StorageReadinessHook{
+		checks:  make(map[string]func() error),
+		timeout: timeout,
+	}
+}
+
+func (h *StorageReadinessHook) RegisterStorage(gvr metav1.GroupVersionResource, storage rest.StorageWithReadiness) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	if _, ok := h.checks[gvr.String()]; !ok {
+		h.checks[gvr.String()] = storage.ReadinessCheck
+	} else {
+		klog.Errorf("Registering storage readiness hook for %s again: ", gvr.String())
+	}
+}
+
+func (h *StorageReadinessHook) check() bool {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	failedChecks := []string{}
+	for gvr, check := range h.checks {
+		if err := check(); err != nil {
+			failedChecks = append(failedChecks, gvr)
+		}
+	}
+	if len(failedChecks) == 0 {
+		klog.Infof("Storage is ready for all registered resources")
+		return true
+	}
+	klog.V(4).Infof("Storage is not ready for: %v", failedChecks)
+	return false
+}
+
+func (h *StorageReadinessHook) Hook(ctx PostStartHookContext) error {
+	deadlineCtx, cancel := context.WithTimeout(ctx, h.timeout)
+	defer cancel()
+	err := wait.PollUntilContextCancel(deadlineCtx, 100*time.Millisecond, true,
+		func(_ context.Context) (bool, error) {
+			if ok := h.check(); ok {
+				return true, nil
+			}
+			return false, nil
+		})
+	if errors.Is(err, context.DeadlineExceeded) {
+		klog.Warningf("Deadline exceeded while waiting for storage readiness... ignoring")
+	}
+	return nil
+}