rebase: bump k8s.io/kubernetes from 1.26.2 to 1.27.2

Bumps [k8s.io/kubernetes](https://github.com/kubernetes/kubernetes) from 1.26.2 to 1.27.2.
- [Release notes](https://github.com/kubernetes/kubernetes/releases)
- [Commits](https://github.com/kubernetes/kubernetes/compare/v1.26.2...v1.27.2)

---
updated-dependencies:
- dependency-name: k8s.io/kubernetes
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

vendor/k8s.io/apiserver/pkg/server/config_selfclient.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net"
+
+	restclient "k8s.io/client-go/rest"
+	netutils "k8s.io/utils/net"
+)
+
+// LoopbackClientServerNameOverride is passed to the apiserver from the loopback client in order to
+// select the loopback certificate via SNI if TLS is used.
+const LoopbackClientServerNameOverride = "apiserver-loopback-client"
+
+func (s *SecureServingInfo) NewClientConfig(caCert []byte) (*restclient.Config, error) {
+	if s == nil || (s.Cert == nil && len(s.SNICerts) == 0) {
+		return nil, nil
+	}
+
+	host, port, err := LoopbackHostPort(s.Listener.Addr().String())
+	if err != nil {
+		return nil, err
+	}
+
+	return &restclient.Config{
+		// Do not limit loopback client QPS.
+		QPS:  -1,
+		Host: "https://" + net.JoinHostPort(host, port),
+		TLSClientConfig: restclient.TLSClientConfig{
+			CAData: caCert,
+		},
+	}, nil
+}
+
+func (s *SecureServingInfo) NewLoopbackClientConfig(token string, loopbackCert []byte) (*restclient.Config, error) {
+	c, err := s.NewClientConfig(loopbackCert)
+	if err != nil || c == nil {
+		return c, err
+	}
+
+	c.BearerToken = token
+	// override the ServerName to select our loopback certificate via SNI. This name is also
+	// used by the client to compare the returns server certificate against.
+	c.TLSClientConfig.ServerName = LoopbackClientServerNameOverride
+
+	return c, nil
+}
+
+// LoopbackHostPort returns the host and port loopback REST clients should use
+// to contact the server.
+func LoopbackHostPort(bindAddress string) (string, string, error) {
+	host, port, err := net.SplitHostPort(bindAddress)
+	if err != nil {
+		// should never happen
+		return "", "", fmt.Errorf("invalid server bind address: %q", bindAddress)
+	}
+
+	isIPv6 := netutils.IsIPv6String(host)
+
+	// Value is expected to be an IP or DNS name, not "0.0.0.0".
+	if host == "0.0.0.0" || host == "::" {
+		// Get ip of local interface, but fall back to "localhost".
+		// Note that "localhost" is resolved with the external nameserver first with Go's stdlib.
+		// So if localhost.<yoursearchdomain> resolves, we don't get a 127.0.0.1 as expected.
+		host = getLoopbackAddress(isIPv6)
+	}
+	return host, port, nil
+}
+
+// getLoopbackAddress returns the ip address of local loopback interface. If any error occurs or loopback interface is not found, will fall back to "localhost"
+func getLoopbackAddress(wantIPv6 bool) string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, address := range addrs {
+			if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && wantIPv6 == netutils.IsIPv6(ipnet.IP) {
+				return ipnet.IP.String()
+			}
+		}
+	}
+	return "localhost"
+}

vendor/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -0,0 +1,201 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	apimachineryversion "k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/klog/v2"
+)
+
+// resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
+type resourceExpirationEvaluator struct {
+	currentMajor int
+	currentMinor int
+	isAlpha      bool
+	// This is usually set for testing for which tests need to be removed.  This prevent insta-failing CI.
+	// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
+	strictRemovedHandlingInAlpha bool
+	// This is usually set by a cluster-admin looking for a short-term escape hatch after something bad happened.
+	// This should be made a flag before merge
+	// Set KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE to prevent removing APIs for one more release.
+	serveRemovedAPIsOneMoreRelease bool
+}
+
+// ResourceExpirationEvaluator indicates whether or not a resource should be served.
+type ResourceExpirationEvaluator interface {
+	// RemoveDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+	// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+	RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage)
+	// ShouldServeForVersion returns true if a particular version cut off is after the current version
+	ShouldServeForVersion(majorRemoved, minorRemoved int) bool
+}
+
+func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
+	ret := &resourceExpirationEvaluator{
+		strictRemovedHandlingInAlpha: false,
+	}
+	if len(currentVersion.Major) > 0 {
+		currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		ret.currentMajor = int(currentMajor64)
+	}
+	if len(currentVersion.Minor) > 0 {
+		// split the "normal" + and - for semver stuff
+		minorString := strings.Split(currentVersion.Minor, "+")[0]
+		minorString = strings.Split(minorString, "-")[0]
+		minorString = strings.Split(minorString, ".")[0]
+		currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		ret.currentMinor = int(currentMinor64)
+	}
+
+	ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
+
+	if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
+		// do nothing
+	} else if envBool, err := strconv.ParseBool(envString); err != nil {
+		return nil, err
+	} else {
+		ret.strictRemovedHandlingInAlpha = envBool
+	}
+
+	if envString, ok := os.LookupEnv("KUBE_APISERVER_SERVE_REMOVED_APIS_FOR_ONE_RELEASE"); !ok {
+		// do nothing
+	} else if envBool, err := strconv.ParseBool(envString); err != nil {
+		return nil, err
+	} else {
+		ret.serveRemovedAPIsOneMoreRelease = envBool
+	}
+
+	return ret, nil
+}
+
+func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versioner runtime.ObjectVersioner, resourceServingInfo rest.Storage) bool {
+	internalPtr := resourceServingInfo.New()
+
+	target := gv
+	// honor storage that overrides group version (used for things like scale subresources)
+	if versionProvider, ok := resourceServingInfo.(rest.GroupVersionKindProvider); ok {
+		target = versionProvider.GroupVersionKind(target).GroupVersion()
+	}
+
+	versionedPtr, err := versioner.ConvertToVersion(internalPtr, target)
+	if err != nil {
+		utilruntime.HandleError(err)
+		return false
+	}
+
+	removed, ok := versionedPtr.(removedInterface)
+	if !ok {
+		return true
+	}
+	majorRemoved, minorRemoved := removed.APILifecycleRemoved()
+	return e.ShouldServeForVersion(majorRemoved, minorRemoved)
+}
+
+func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
+	if e.currentMajor < majorRemoved {
+		return true
+	}
+	if e.currentMajor > majorRemoved {
+		return false
+	}
+	if e.currentMinor < minorRemoved {
+		return true
+	}
+	if e.currentMinor > minorRemoved {
+		return false
+	}
+	// at this point major and minor are equal, so this API should be removed when the current release GAs.
+	// If this is an alpha tag, don't remove by default, but allow the option.
+	// If the cluster-admin has requested serving one more release, allow it.
+	if e.isAlpha && e.strictRemovedHandlingInAlpha { // don't serve in alpha if we want strict handling
+		return false
+	}
+	if e.isAlpha { // alphas are allowed to continue serving expired betas while we clean up the test
+		return true
+	}
+	if e.serveRemovedAPIsOneMoreRelease { // cluster-admins are allowed to kick the can one release down the road
+		return true
+	}
+	return false
+}
+
+type removedInterface interface {
+	APILifecycleRemoved() (major, minor int)
+}
+
+// removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
+// versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
+func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
+	versionsToRemove := sets.NewString()
+	for apiVersion := range sets.StringKeySet(versionedResourcesStorageMap) {
+		versionToResource := versionedResourcesStorageMap[apiVersion]
+		resourcesToRemove := sets.NewString()
+		for resourceName, resourceServingInfo := range versionToResource {
+			if !e.shouldServe(schema.GroupVersion{Group: groupName, Version: apiVersion}, versioner, resourceServingInfo) {
+				resourcesToRemove.Insert(resourceName)
+			}
+		}
+
+		for resourceName := range versionedResourcesStorageMap[apiVersion] {
+			if !shouldRemoveResourceAndSubresources(resourcesToRemove, resourceName) {
+				continue
+			}
+
+			klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
+			delete(versionToResource, resourceName)
+		}
+		versionedResourcesStorageMap[apiVersion] = versionToResource
+
+		if len(versionedResourcesStorageMap[apiVersion]) == 0 {
+			versionsToRemove.Insert(apiVersion)
+		}
+	}
+
+	for _, apiVersion := range versionsToRemove.List() {
+		klog.V(1).Infof("Removing version %v.%v because it is time to stop serving it because it has no resources per APILifecycle.", apiVersion, groupName)
+		delete(versionedResourcesStorageMap, apiVersion)
+	}
+}
+
+func shouldRemoveResourceAndSubresources(resourcesToRemove sets.String, resourceName string) bool {
+	for _, resourceToRemove := range resourcesToRemove.List() {
+		if resourceName == resourceToRemove {
+			return true
+		}
+		// our API works on nesting, so you can have deployments, deployments/status, and deployments/scale.  Not all subresources
+		// serve the parent type, but if the parent type (deployments in this case), has been removed, it's subresources should be removed too.
+		if strings.HasPrefix(resourceName, resourceToRemove+"/") {
+			return true
+		}
+	}
+	return false
+}

vendor/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/client-go/rest"
+)
+
+// DeprecatedInsecureServingInfo is the main context object for the insecure http server.
+// HTTP does NOT include authentication or authorization.
+// You shouldn't be using this.  It makes sig-auth sad.
+type DeprecatedInsecureServingInfo struct {
+	// Listener is the secure server network listener.
+	Listener net.Listener
+	// optional server name for log messages
+	Name string
+}
+
+// Serve starts an insecure http server with the given handler. It fails only if
+// the initial listen call fails. It does not block.
+func (s *DeprecatedInsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error {
+	insecureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+
+		IdleTimeout:       90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+		ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound
+	}
+
+	if len(s.Name) > 0 {
+		klog.Infof("Serving %s insecurely on %s", s.Name, s.Listener.Addr())
+	} else {
+		klog.Infof("Serving insecurely on %s", s.Listener.Addr())
+	}
+	_, _, err := RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
+	// NOTE: we do not handle stoppedCh returned by RunServer for graceful termination here
+	return err
+}
+
+func (s *DeprecatedInsecureServingInfo) NewLoopbackClientConfig() (*rest.Config, error) {
+	if s == nil {
+		return nil, nil
+	}
+
+	host, port, err := LoopbackHostPort(s.Listener.Addr().String())
+	if err != nil {
+		return nil, err
+	}
+
+	return &rest.Config{
+		Host: "http://" + net.JoinHostPort(host, port),
+		// Increase QPS limits. The client is currently passed to all admission plugins,
+		// and those can be throttled in case of higher load on apiserver - see #22340 and #22422
+		// for more details. Once #22422 is fixed, we may want to remove it.
+		QPS:   50,
+		Burst: 100,
+	}, nil
+}
+
+// InsecureSuperuser implements authenticator.Request to always return a superuser.
+// This is functionally equivalent to skipping authentication and authorization,
+// but allows apiserver code to stop special-casing a nil user to skip authorization checks.
+type InsecureSuperuser struct{}
+
+func (InsecureSuperuser) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) {
+	auds, _ := authenticator.AudiencesFrom(req.Context())
+	return &authenticator.Response{
+		User: &user.DefaultInfo{
+			Name:   "system:unsecured",
+			Groups: []string{user.SystemPrivilegedGroup, user.AllAuthenticated},
+		},
+		Audiences: auds,
+	}, true, nil
+}

vendor/k8s.io/apiserver/pkg/server/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package server contains the plumbing to create kubernetes-like API server command.
+package server // import "k8s.io/apiserver/pkg/server"

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// certKeyContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}
+
+// sniCertKeyContent holds the content for the cert and key as well as any explicit names
+type sniCertKeyContent struct {
+	certKeyContent
+	sniNames []string
+}
+
+func (c *sniCertKeyContent) Equal(rhs *sniCertKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if len(c.sniNames) != len(rhs.sniNames) {
+		return false
+	}
+
+	for i := range c.sniNames {
+		if c.sniNames[i] != rhs.sniNames[i] {
+			return false
+		}
+	}
+
+	return c.certKeyContent.Equal(&rhs.certKeyContent)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// dynamicCertificateContent holds the content that overrides the baseTLSConfig
+type dynamicCertificateContent struct {
+	// clientCA holds the content for the clientCA bundle
+	clientCA    caBundleContent
+	servingCert certKeyContent
+	sniCerts    []sniCertKeyContent
+}
+
+// caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
+// method receiver.
+type caBundleContent struct {
+	caBundle []byte
+}
+
+func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if !c.clientCA.Equal(&rhs.clientCA) {
+		return false
+	}
+
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
+	if len(c.sniCerts) != len(rhs.sniCerts) {
+		return false
+	}
+
+	for i := range c.sniCerts {
+		if !c.sniCerts[i].Equal(&rhs.sniCerts[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.caBundle, rhs.caBundle)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -0,0 +1,274 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes
+// It also fulfills the authenticator interface to provide verifyoptions
+type ConfigMapCAController struct {
+	name string
+
+	configmapLister    corev1listers.ConfigMapLister
+	configmapNamespace string
+	configmapName      string
+	configmapKey       string
+	// configMapInformer is tracked so that we can start these on Run
+	configMapInformer cache.SharedIndexInformer
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	queue workqueue.RateLimitingInterface
+	// preRunCaches are the caches to sync before starting the work of this control loop
+	preRunCaches []cache.InformerSynced
+}
+
+var _ CAContentProvider = &ConfigMapCAController{}
+var _ ControllerRunner = &ConfigMapCAController{}
+
+// NewDynamicCAFromConfigMapController returns a CAContentProvider based on a configmap that automatically reloads content.
+// It is near-realtime via an informer.
+func NewDynamicCAFromConfigMapController(purpose, namespace, name, key string, kubeClient kubernetes.Interface) (*ConfigMapCAController, error) {
+	if len(purpose) == 0 {
+		return nil, fmt.Errorf("missing purpose for ca bundle")
+	}
+	if len(namespace) == 0 {
+		return nil, fmt.Errorf("missing namespace for ca bundle")
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("missing name for ca bundle")
+	}
+	if len(key) == 0 {
+		return nil, fmt.Errorf("missing key for ca bundle")
+	}
+	caContentName := fmt.Sprintf("%s::%s::%s::%s", purpose, namespace, name, key)
+
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	uncastConfigmapInformer := corev1informers.NewFilteredConfigMapInformer(kubeClient, namespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *v1.ListOptions) {
+		listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String()
+	})
+
+	configmapLister := corev1listers.NewConfigMapLister(uncastConfigmapInformer.GetIndexer())
+
+	c := &ConfigMapCAController{
+		name:               caContentName,
+		configmapNamespace: namespace,
+		configmapName:      name,
+		configmapKey:       key,
+		configmapLister:    configmapLister,
+		configMapInformer:  uncastConfigmapInformer,
+
+		queue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)),
+		preRunCaches: []cache.InformerSynced{uncastConfigmapInformer.HasSynced},
+	}
+
+	uncastConfigmapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+		},
+	})
+
+	return c, nil
+}
+
+func (c *ConfigMapCAController) keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return c.configmapNamespace + "/" + c.configmapName
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *ConfigMapCAController) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *ConfigMapCAController) loadCABundle() error {
+	configMap, err := c.configmapLister.ConfigMaps(c.configmapNamespace).Get(c.configmapName)
+	if err != nil {
+		return err
+	}
+	caBundle := configMap.Data[c.configmapKey]
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged([]byte(caBundle)) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), []byte(caBundle))
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *ConfigMapCAController) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *ConfigMapCAController) RunOnce(ctx context.Context) error {
+	// Ignore the error when running once because when using a dynamically loaded ca file, because we think it's better to have nothing for
+	// a brief time than completely crash.  If crashing is necessary, higher order logic like a healthcheck and cause failures.
+	_ = c.loadCABundle()
+	return nil
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *ConfigMapCAController) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// we have a personal informer that is narrowly scoped, start it.
+	go c.configMapInformer.Run(ctx.Done())
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync(c.name, ctx.Done(), c.preRunCaches...) {
+		return
+	}
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *ConfigMapCAController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *ConfigMapCAController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *ConfigMapCAController) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *ConfigMapCAController) CurrentCABundleContent() []byte {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return nil // this can happen if we've been unable load data from the apiserver for some reason
+	}
+
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *ConfigMapCAController) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		// This can happen if we've been unable load data from the apiserver for some reason.
+		// In this case, we should not accept any connections on the basis of this ca bundle.
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -0,0 +1,290 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	"k8s.io/client-go/util/cert"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce(ctx context.Context) error
+
+	// Run should be called a go .Run
+	Run(ctx context.Context, workers int)
+}
+
+// DynamicFileCAContent provides a CAContentProvider that can dynamically react to new file content
+// It also fulfills the authenticator interface to provide verifyoptions
+type DynamicFileCAContent struct {
+	name string
+
+	// filename is the name the file to read.
+	filename string
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
+type caBundleAndVerifier struct {
+	caBundle      []byte
+	verifyOptions x509.VerifyOptions
+}
+
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
+func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+	name := fmt.Sprintf("%s::%s", purpose, filename)
+
+	ret := &DynamicFileCAContent{
+		name:     name,
+		filename: filename,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadCABundle(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *DynamicFileCAContent) loadCABundle() error {
+	caBundle, err := ioutil.ReadFile(c.filename)
+	if err != nil {
+		return err
+	}
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged(caBundle) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle)
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+	klog.V(2).InfoS("Loaded a new CA Bundle and Verifier", "name", c.Name())
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce(ctx context.Context) error {
+	return c.loadCABundle()
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (c *DynamicFileCAContent) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start the loop that watches the CA file until stopCh is closed.
+	go wait.Until(func() {
+		if err := c.watchCAFile(ctx.Done()); err != nil {
+			klog.ErrorS(err, "Failed to watch CA file, will retry later")
+		}
+	}, time.Minute, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *DynamicFileCAContent) watchCAFile(stopCh <-chan struct{}) error {
+	// Trigger a check here to ensure the content will be checked periodically even if the following watch fails.
+	c.queue.Add(workItemKey)
+
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return fmt.Errorf("error creating fsnotify watcher: %v", err)
+	}
+	defer w.Close()
+
+	if err = w.Add(c.filename); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
+	}
+	// Trigger a check in case the file is updated before the watch starts.
+	c.queue.Add(workItemKey)
+
+	for {
+		select {
+		case e := <-w.Events:
+			if err := c.handleWatchEvent(e, w); err != nil {
+				return err
+			}
+		case err := <-w.Errors:
+			return fmt.Errorf("received fsnotify error: %v", err)
+		case <-stopCh:
+			return nil
+		}
+	}
+}
+
+// handleWatchEvent triggers reloading the CA file, and restarts a new watch if it's a Remove or Rename event.
+func (c *DynamicFileCAContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
+	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
+	defer c.queue.Add(workItemKey)
+	if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
+		return nil
+	}
+	if err := w.Remove(c.filename); err != nil {
+		klog.InfoS("Failed to remove file watch, it may have been deleted", "file", c.filename, "err", err)
+	}
+	if err := w.Add(c.filename); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.filename, err)
+	}
+	return nil
+}
+
+func (c *DynamicFileCAContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileCAContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}
+
+// newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.
+// It will return a nil function if you pass an empty CA file.
+func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) {
+	if len(caBundle) == 0 {
+		return nil, fmt.Errorf("missing content for CA bundle %q", name)
+	}
+
+	// Wrap with an x509 verifier
+	var err error
+	verifyOptions := defaultVerifyOptions()
+	verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle)
+	if err != nil {
+		return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err)
+	}
+
+	return &caBundleAndVerifier{
+		caBundle:      caBundle,
+		verifyOptions: verifyOptions,
+	}, nil
+}
+
+// defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time,
+// and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth)
+func defaultVerifyOptions() x509.VerifyOptions {
+	return x509.VerifyOptions{
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -0,0 +1,233 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// DynamicCertKeyPairContent provides a CertKeyContentProvider that can dynamically react to new file content
+type DynamicCertKeyPairContent struct {
+	name string
+
+	// certFile is the name of the certificate file to read.
+	certFile string
+	// keyFile is the name of the key file to read.
+	keyFile string
+
+	// certKeyPair is a certKeyContent that contains the last read, non-zero length content of the key and cert
+	certKeyPair atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ CertKeyContentProvider = &DynamicCertKeyPairContent{}
+var _ ControllerRunner = &DynamicCertKeyPairContent{}
+
+// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename
+func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicCertKeyPairContent, error) {
+	if len(certFile) == 0 || len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for serving cert")
+	}
+	name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile)
+
+	ret := &DynamicCertKeyPairContent{
+		name:     name,
+		certFile: certFile,
+		keyFile:  keyFile,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadCertKeyPair(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the serving cert content changes.
+func (c *DynamicCertKeyPairContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCertKeyPair determines the next set of content for the file.
+func (c *DynamicCertKeyPairContent) loadCertKeyPair() error {
+	cert, err := ioutil.ReadFile(c.certFile)
+	if err != nil {
+		return err
+	}
+	key, err := ioutil.ReadFile(c.keyFile)
+	if err != nil {
+		return err
+	}
+	if len(cert) == 0 || len(key) == 0 {
+		return fmt.Errorf("missing content for serving cert %q", c.Name())
+	}
+
+	// Ensure that the key matches the cert and both are valid
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return err
+	}
+
+	newCertKey := &certKeyContent{
+		cert: cert,
+		key:  key,
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := c.certKeyPair.Load().(*certKeyContent)
+	if ok && existing != nil && existing.Equal(newCertKey) {
+		return nil
+	}
+
+	c.certKeyPair.Store(newCertKey)
+	klog.V(2).InfoS("Loaded a new cert/key pair", "name", c.Name())
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicCertKeyPairContent) RunOnce(ctx context.Context) error {
+	return c.loadCertKeyPair()
+}
+
+// Run starts the controller and blocks until context is killed.
+func (c *DynamicCertKeyPairContent) Run(ctx context.Context, workers int) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", c.name)
+	defer klog.InfoS("Shutting down controller", "name", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, ctx.Done())
+
+	// start the loop that watches the cert and key files until stopCh is closed.
+	go wait.Until(func() {
+		if err := c.watchCertKeyFile(ctx.Done()); err != nil {
+			klog.ErrorS(err, "Failed to watch cert and key file, will retry later")
+		}
+	}, time.Minute, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (c *DynamicCertKeyPairContent) watchCertKeyFile(stopCh <-chan struct{}) error {
+	// Trigger a check here to ensure the content will be checked periodically even if the following watch fails.
+	c.queue.Add(workItemKey)
+
+	w, err := fsnotify.NewWatcher()
+	if err != nil {
+		return fmt.Errorf("error creating fsnotify watcher: %v", err)
+	}
+	defer w.Close()
+
+	if err := w.Add(c.certFile); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.certFile, err)
+	}
+	if err := w.Add(c.keyFile); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", c.keyFile, err)
+	}
+	// Trigger a check in case the file is updated before the watch starts.
+	c.queue.Add(workItemKey)
+
+	for {
+		select {
+		case e := <-w.Events:
+			if err := c.handleWatchEvent(e, w); err != nil {
+				return err
+			}
+		case err := <-w.Errors:
+			return fmt.Errorf("received fsnotify error: %v", err)
+		case <-stopCh:
+			return nil
+		}
+	}
+}
+
+// handleWatchEvent triggers reloading the cert and key file, and restarts a new watch if it's a Remove or Rename event.
+// If one file is updated before the other, the loadCertKeyPair method will catch the mismatch and will not apply the
+// change. When an event of the other file is received, it will trigger reloading the files again and the new content
+// will be loaded and used.
+func (c *DynamicCertKeyPairContent) handleWatchEvent(e fsnotify.Event, w *fsnotify.Watcher) error {
+	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
+	defer c.queue.Add(workItemKey)
+	if !e.Has(fsnotify.Remove) && !e.Has(fsnotify.Rename) {
+		return nil
+	}
+	if err := w.Remove(e.Name); err != nil {
+		klog.InfoS("Failed to remove file watch, it may have been deleted", "file", e.Name, "err", err)
+	}
+	if err := w.Add(e.Name); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %v", e.Name, err)
+	}
+	return nil
+}
+
+func (c *DynamicCertKeyPairContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicCertKeyPairContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCertKeyPair()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicCertKeyPairContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key byte content
+func (c *DynamicCertKeyPairContent) CurrentCertKeyContent() ([]byte, []byte) {
+	certKeyContent := c.certKeyPair.Load().(*certKeyContent)
+	return certKeyContent.cert, certKeyContent.key
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content
+type DynamicFileSNIContent struct {
+	*DynamicCertKeyPairContent
+	sniNames []string
+}
+
+var _ SNICertKeyContentProvider = &DynamicFileSNIContent{}
+var _ ControllerRunner = &DynamicFileSNIContent{}
+
+// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names
+func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) {
+	servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &DynamicFileSNIContent{
+		DynamicCertKeyPairContent: servingContent,
+		sniNames:                  sniNames,
+	}
+	if err := ret.loadCertKeyPair(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// SNINames returns explicitly set SNI names for the certificate. These are not dynamic.
+func (c *DynamicFileSNIContent) SNINames() []string {
+	return c.sniNames
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/interfaces.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+)
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
+	Enqueue()
+}
+
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes.
+	// This is a noop on static providers.
+	AddListener(listener Listener)
+}
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	Notifier
+
+	// Name is just an identifier.
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content. Errors can be
+	// contained to the controllers initializing the value. By the time you get
+	// here, you should always be returning a value that won't fail.
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators.
+	VerifyOptions() (x509.VerifyOptions, bool)
+}
+
+// CertKeyContentProvider provides a certificate and matching private key.
+type CertKeyContentProvider interface {
+	Notifier
+
+	// Name is just an identifier.
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content.
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// SNICertKeyContentProvider provides a certificate and matching private key as
+// well as optional explicit names.
+type SNICertKeyContentProvider interface {
+	Notifier
+
+	CertKeyContentProvider
+	// SNINames provides names used for SNI. May return nil.
+	SNINames() []string
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/named_certificates.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+)
+
+// BuildNamedCertificates returns a map of *tls.Certificate by name. It's
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// is invalid. Returns nil if len(certs) == 0
+func (c *DynamicServingCertificateController) BuildNamedCertificates(sniCerts []sniCertKeyContent) (map[string]*tls.Certificate, error) {
+	nameToCertificate := map[string]*tls.Certificate{}
+	byNameExplicit := map[string]*tls.Certificate{}
+
+	// Iterate backwards so that earlier certs take precedence in the names map
+	for i := len(sniCerts) - 1; i >= 0; i-- {
+		cert, err := tls.X509KeyPair(sniCerts[i].cert, sniCerts[i].key)
+		if err != nil {
+			return nil, fmt.Errorf("invalid SNI cert keypair [%d/%q]: %v", i, c.sniCerts[i].Name(), err)
+		}
+
+		// error is not possible given above call to X509KeyPair
+		x509Cert, _ := x509.ParseCertificate(cert.Certificate[0])
+
+		names := sniCerts[i].sniNames
+		for _, name := range names {
+			byNameExplicit[name] = &cert
+		}
+
+		klog.V(2).InfoS("Loaded SNI cert", "index", i, "certName", c.sniCerts[i].Name(), "certDetail", GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.sniCerts[i].Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "SNICertificateReload", "loaded SNI cert [%d/%q]: %s with explicit names %v", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert), names)
+		}
+
+		if len(names) == 0 {
+			names = getCertificateNames(x509Cert)
+			for _, name := range names {
+				nameToCertificate[name] = &cert
+			}
+		}
+	}
+
+	// Explicitly set names must override
+	for k, v := range byNameExplicit {
+		nameToCertificate[k] = v
+	}
+
+	return nameToCertificate, nil
+}
+
+// getCertificateNames returns names for an x509.Certificate. The names are
+// suitable for use in tls.Config#NamedCertificates.
+func getCertificateNames(cert *x509.Certificate) []string {
+	var names []string
+
+	cn := cert.Subject.CommonName
+	cnIsIP := netutils.ParseIPSloppy(cn) != nil
+	cnIsValidDomain := cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0
+	// don't use the CN if it is a valid IP because our IP serving detection may unexpectedly use it to terminate the connection.
+	if !cnIsIP && cnIsValidDomain {
+		names = append(names, cn)
+	}
+	names = append(names, cert.DNSNames...)
+	// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
+	// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
+
+	return names
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+)
+
+type staticCAContent struct {
+	name     string
+	caBundle *caBundleAndVerifier
+}
+
+var _ CAContentProvider = &staticCAContent{}
+
+// NewStaticCAContent returns a CAContentProvider that always returns the same value
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCAContent{
+		name:     name,
+		caBundle: caBundleAndVerifier,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCAContent) Name() string {
+	return c.name
+}
+
+func (c *staticCAContent) AddListener(Listener) {}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	return c.caBundle.verifyOptions, true
+}
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+func (c *staticCertKeyContent) AddListener(Listener) {}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}
+
+type staticSNICertKeyContent struct {
+	staticCertKeyContent
+	sniNames []string
+}
+
+// NewStaticSNICertKeyContent returns a SNICertKeyContentProvider that always returns the same value
+func NewStaticSNICertKeyContent(name string, cert, key []byte, sniNames ...string) (SNICertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticSNICertKeyContent{
+		staticCertKeyContent: staticCertKeyContent{
+			name: name,
+			cert: cert,
+			key:  key,
+		},
+		sniNames: sniNames,
+	}, nil
+}
+
+func (c *staticSNICertKeyContent) SNINames() []string {
+	return c.sniNames
+}
+
+func (c *staticSNICertKeyContent) AddListener(Listener) {}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -0,0 +1,284 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+const workItemKey = "key"
+
+// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
+type DynamicServingCertificateController struct {
+	// baseTLSConfig is the static portion of the tlsConfig for serving to clients.  It is copied and the copy is mutated
+	// based on the dynamic cert state.
+	baseTLSConfig *tls.Config
+
+	// clientCA provides the very latest content of the ca bundle
+	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
+	// sniCerts are a list of CertKeyContentProvider with associated names used for SNI
+	sniCerts []SNICertKeyContentProvider
+
+	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
+	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
+	currentlyServedContent *dynamicCertificateContent
+	// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
+	currentServingTLSConfig atomic.Value
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue         workqueue.RateLimitingInterface
+	eventRecorder events.EventRecorder
+}
+
+var _ Listener = &DynamicServingCertificateController{}
+
+// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
+func NewDynamicServingCertificateController(
+	baseTLSConfig *tls.Config,
+	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
+	sniCerts []SNICertKeyContentProvider,
+	eventRecorder events.EventRecorder,
+) *DynamicServingCertificateController {
+	c := &DynamicServingCertificateController{
+		baseTLSConfig: baseTLSConfig,
+		clientCA:      clientCA,
+		servingCert:   servingCert,
+		sniCerts:      sniCerts,
+
+		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
+		eventRecorder: eventRecorder,
+	}
+
+	return c
+}
+
+// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
+func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+	uncastObj := c.currentServingTLSConfig.Load()
+	if uncastObj == nil {
+		return nil, errors.New("dynamiccertificates: configuration not ready")
+	}
+	tlsConfig, ok := uncastObj.(*tls.Config)
+	if !ok {
+		return nil, errors.New("dynamiccertificates: unexpected config type")
+	}
+
+	tlsConfigCopy := tlsConfig.Clone()
+
+	// if the client set SNI information, just use our "normal" SNI flow
+	if len(clientHello.ServerName) > 0 {
+		return tlsConfigCopy, nil
+	}
+
+	// if the client didn't set SNI, then we need to inspect the requested IP so that we can choose
+	// a certificate from our list if we specifically handle that IP.  This can happen when an IP is specifically mapped by name.
+	host, _, err := net.SplitHostPort(clientHello.Conn.LocalAddr().String())
+	if err != nil {
+		return tlsConfigCopy, nil
+	}
+
+	ipCert, ok := tlsConfigCopy.NameToCertificate[host]
+	if !ok {
+		return tlsConfigCopy, nil
+	}
+	tlsConfigCopy.Certificates = []tls.Certificate{*ipCert}
+	tlsConfigCopy.NameToCertificate = nil
+
+	return tlsConfigCopy, nil
+}
+
+// newTLSContent determines the next set of content for overriding the baseTLSConfig.
+func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
+	newContent := &dynamicCertificateContent{}
+
+	if c.clientCA != nil {
+		currClientCABundle := c.clientCA.CurrentCABundleContent()
+		// we allow removing all client ca bundles because the server is still secure when this happens. it just means
+		// that there isn't a hint to clients about which client-cert to used.  this happens when there is no client-ca
+		// yet known for authentication, which can happen in aggregated apiservers and some kube-apiserver deployment modes.
+		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
+	}
+
+	for i, sniCert := range c.sniCerts {
+		currCert, currKey := sniCert.CurrentCertKeyContent()
+		if len(currCert) == 0 || len(currKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty SNI certificate from %d/%q", i, sniCert.Name())
+		}
+
+		newContent.sniCerts = append(newContent.sniCerts, sniCertKeyContent{certKeyContent: certKeyContent{cert: currCert, key: currKey}, sniNames: sniCert.SNINames()})
+	}
+
+	return newContent, nil
+}
+
+// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
+// GetConfigForClient.
+func (c *DynamicServingCertificateController) syncCerts() error {
+	newContent, err := c.newTLSContent()
+	if err != nil {
+		return err
+	}
+	// if the content is the same as what we currently have, we can simply skip it.  This works because we are single
+	// threaded.  If you ever make this multi-threaded, add a lock.
+	if newContent.Equal(c.currentlyServedContent) {
+		return nil
+	}
+
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
+	// parse new content to add to TLSConfig
+	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
+		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file %q: %v", string(newContent.clientCA.caBundle), err)
+		}
+		for i, cert := range newClientCAs {
+			klog.V(2).InfoS("Loaded client CA", "index", i, "certName", c.clientCA.Name(), "certDetail", GetHumanCertDetail(cert))
+			if c.eventRecorder != nil {
+				c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.clientCA.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			}
+
+			newClientCAPool.AddCert(cert)
+		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
+	}
+
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).InfoS("Loaded serving cert", "certName", c.servingCert.Name(), "certDetail", GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(&corev1.ObjectReference{Name: c.servingCert.Name()}, nil, corev1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+	}
+
+	if len(newContent.sniCerts) > 0 {
+		newTLSConfigCopy.NameToCertificate, err = c.BuildNamedCertificates(newContent.sniCerts)
+		if err != nil {
+			return fmt.Errorf("unable to build named certificate map: %v", err)
+		}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if servingCert is not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, sniCert := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *sniCert)
+		}
+	}
+
+	// store new values of content for serving.
+	c.currentServingTLSConfig.Store(newTLSConfigCopy)
+	c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
+
+	return nil
+}
+
+// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
+func (c *DynamicServingCertificateController) RunOnce() error {
+	return c.syncCerts()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.InfoS("Starting DynamicServingCertificateController")
+	defer klog.InfoS("Shutting down DynamicServingCertificateController")
+
+	// synchronously load once.  We will trigger again, so ignoring any error is fine
+	_ = c.RunOnce()
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.Until(func() {
+		c.Enqueue()
+	}, 1*time.Minute, stopCh)
+
+	<-stopCh
+}
+
+func (c *DynamicServingCertificateController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicServingCertificateController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.syncCerts()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
+func (c *DynamicServingCertificateController) Enqueue() {
+	c.queue.Add(workItemKey)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/x509"
+	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+type unionCAContent []CAContentProvider
+
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
+// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
+func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
+	return unionCAContent(caContentProviders)
+}
+
+// Name is just an identifier
+func (c unionCAContent) Name() string {
+	names := []string{}
+	for _, curr := range c {
+		names = append(names, curr.Name())
+	}
+	return strings.Join(names, ",")
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) CurrentCABundleContent() []byte {
+	caBundles := [][]byte{}
+	for _, curr := range c {
+		if currCABytes := curr.CurrentCABundleContent(); len(currCABytes) > 0 {
+			caBundles = append(caBundles, []byte(strings.TrimSpace(string(currCABytes))))
+		}
+	}
+
+	return bytes.Join(caBundles, []byte("\n"))
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	currCABundle := c.CurrentCABundleContent()
+	if len(currCABundle) == 0 {
+		return x509.VerifyOptions{}, false
+	}
+
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions, true
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		curr.AddListener(listener)
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce(ctx context.Context) error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(ctx); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(ctx context.Context, workers int) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(ctx, workers)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
+// kube-apiserver usage of certs.
+func GetHumanCertDetail(certificate *x509.Certificate) string {
+	humanName := certificate.Subject.CommonName
+	signerHumanName := certificate.Issuer.CommonName
+	if certificate.Subject.CommonName == certificate.Issuer.CommonName {
+		signerHumanName = "<self>"
+	}
+
+	usages := []string{}
+	for _, curr := range certificate.ExtKeyUsage {
+		if curr == x509.ExtKeyUsageClientAuth {
+			usages = append(usages, "client")
+			continue
+		}
+		if curr == x509.ExtKeyUsageServerAuth {
+			usages = append(usages, "serving")
+			continue
+		}
+
+		usages = append(usages, fmt.Sprintf("%d", curr))
+	}
+
+	validServingNames := []string{}
+	for _, ip := range certificate.IPAddresses {
+		validServingNames = append(validServingNames, ip.String())
+	}
+	validServingNames = append(validServingNames, certificate.DNSNames...)
+	servingString := ""
+	if len(validServingNames) > 0 {
+		servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
+	}
+
+	groupString := ""
+	if len(certificate.Subject.Organization) > 0 {
+		groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
+	}
+
+	return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
+		time.Now().UTC())
+}

vendor/k8s.io/apiserver/pkg/server/filters/OWNERS

@@ -0,0 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts
+  - dims

vendor/k8s.io/apiserver/pkg/server/filters/content_type.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import "net/http"
+
+// WithContentType sets both the Content-Type and the X-Content-Type-Options (nosniff) header
+func WithContentType(handler http.Handler, contentType string) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", contentType)
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		handler.ServeHTTP(w, r)
+	})
+}

vendor/k8s.io/apiserver/pkg/server/filters/cors.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"regexp"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+// TODO: use restful.CrossOriginResourceSharing
+// See github.com/emicklei/go-restful/blob/master/examples/cors/restful-CORS-filter.go, and
+// github.com/emicklei/go-restful/blob/master/examples/basicauth/restful-basic-authentication.go
+// Or, for a more detailed implementation use https://github.com/martini-contrib/cors
+// or implement CORS at your proxy layer.
+
+// WithCORS is a simple CORS implementation that wraps an http Handler.
+// Pass nil for allowedMethods and allowedHeaders to use the defaults. If allowedOriginPatterns
+// is empty or nil, no CORS support is installed.
+func WithCORS(handler http.Handler, allowedOriginPatterns []string, allowedMethods []string, allowedHeaders []string, exposedHeaders []string, allowCredentials string) http.Handler {
+	if len(allowedOriginPatterns) == 0 {
+		return handler
+	}
+	allowedOriginPatternsREs := allowedOriginRegexps(allowedOriginPatterns)
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		origin := req.Header.Get("Origin")
+		if origin != "" {
+			allowed := false
+			for _, re := range allowedOriginPatternsREs {
+				if allowed = re.MatchString(origin); allowed {
+					break
+				}
+			}
+			if allowed {
+				w.Header().Set("Access-Control-Allow-Origin", origin)
+				// Set defaults for methods and headers if nothing was passed
+				if allowedMethods == nil {
+					allowedMethods = []string{"POST", "GET", "OPTIONS", "PUT", "DELETE", "PATCH"}
+				}
+				if allowedHeaders == nil {
+					allowedHeaders = []string{"Content-Type", "Content-Length", "Accept-Encoding", "X-CSRF-Token", "Authorization", "X-Requested-With", "If-Modified-Since"}
+				}
+				if exposedHeaders == nil {
+					exposedHeaders = []string{"Date"}
+				}
+				w.Header().Set("Access-Control-Allow-Methods", strings.Join(allowedMethods, ", "))
+				w.Header().Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
+				w.Header().Set("Access-Control-Expose-Headers", strings.Join(exposedHeaders, ", "))
+				w.Header().Set("Access-Control-Allow-Credentials", allowCredentials)
+
+				// Stop here if its a preflight OPTIONS request
+				if req.Method == "OPTIONS" {
+					w.WriteHeader(http.StatusNoContent)
+					return
+				}
+			}
+		}
+		// Dispatch to the next handler
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func allowedOriginRegexps(allowedOrigins []string) []*regexp.Regexp {
+	res, err := compileRegexps(allowedOrigins)
+	if err != nil {
+		klog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(allowedOrigins, ","), err)
+	}
+	return res
+}
+
+// Takes a list of strings and compiles them into a list of regular expressions
+func compileRegexps(regexpStrings []string) ([]*regexp.Regexp, error) {
+	regexps := []*regexp.Regexp{}
+	for _, regexpStr := range regexpStrings {
+		r, err := regexp.Compile(regexpStr)
+		if err != nil {
+			return []*regexp.Regexp{}, err
+		}
+		regexps = append(regexps, r)
+	}
+	return regexps, nil
+}

vendor/k8s.io/apiserver/pkg/server/filters/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package filters contains all the http handler chain filters which
+// are not api related.
+package filters // import "k8s.io/apiserver/pkg/server/filters"

vendor/k8s.io/apiserver/pkg/server/filters/goaway.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"math/rand"
+	"net/http"
+	"sync"
+)
+
+// GoawayDecider decides if server should send a GOAWAY
+type GoawayDecider interface {
+	Goaway(r *http.Request) bool
+}
+
+var (
+	// randPool used to get a rand.Rand and generate a random number thread-safely,
+	// which improve the performance of using rand.Rand with a locker
+	randPool = &sync.Pool{
+		New: func() interface{} {
+			return rand.New(rand.NewSource(rand.Int63()))
+		},
+	}
+)
+
+// WithProbabilisticGoaway returns an http.Handler that send GOAWAY probabilistically
+// according to the given chance for HTTP2 requests. After client receive GOAWAY,
+// the in-flight long-running requests will not be influenced, and the new requests
+// will use a new TCP connection to re-balancing to another server behind the load balance.
+func WithProbabilisticGoaway(inner http.Handler, chance float64) http.Handler {
+	return &goaway{
+		handler: inner,
+		decider: &probabilisticGoawayDecider{
+			chance: chance,
+			next: func() float64 {
+				rnd := randPool.Get().(*rand.Rand)
+				ret := rnd.Float64()
+				randPool.Put(rnd)
+				return ret
+			},
+		},
+	}
+}
+
+// goaway send a GOAWAY to client according to decider for HTTP2 requests
+type goaway struct {
+	handler http.Handler
+	decider GoawayDecider
+}
+
+// ServeHTTP implement HTTP handler
+func (h *goaway) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Proto == "HTTP/2.0" && h.decider.Goaway(r) {
+		// Send a GOAWAY and tear down the TCP connection when idle.
+		w.Header().Set("Connection", "close")
+	}
+
+	h.handler.ServeHTTP(w, r)
+}
+
+// probabilisticGoawayDecider send GOAWAY probabilistically according to chance
+type probabilisticGoawayDecider struct {
+	chance float64
+	next   func() float64
+}
+
+// Goaway implement GoawayDecider
+func (p *probabilisticGoawayDecider) Goaway(r *http.Request) bool {
+	return p.next() < p.chance
+}

vendor/k8s.io/apiserver/pkg/server/filters/hsts.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+)
+
+// WithHSTS is a simple HSTS implementation that wraps an http Handler.
+// If hstsDirectives is empty or nil, no HSTS support is installed.
+func WithHSTS(handler http.Handler, hstsDirectives []string) http.Handler {
+	if len(hstsDirectives) == 0 {
+		return handler
+	}
+	allDirectives := strings.Join(hstsDirectives, "; ")
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		// Chrome and Mozilla Firefox maintain an HSTS preload list
+		// issue : golang.org/issue/26162
+		// Set the Strict-Transport-Security header if it is not already set
+		if _, ok := w.Header()["Strict-Transport-Security"]; !ok {
+			w.Header().Set("Strict-Transport-Security", allDirectives)
+		}
+		handler.ServeHTTP(w, req)
+	})
+}

vendor/k8s.io/apiserver/pkg/server/filters/longrunning.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// BasicLongRunningRequestCheck returns true if the given request has one of the specified verbs or one of the specified subresources, or is a profiler request.
+func BasicLongRunningRequestCheck(longRunningVerbs, longRunningSubresources sets.String) apirequest.LongRunningRequestCheck {
+	return func(r *http.Request, requestInfo *apirequest.RequestInfo) bool {
+		if longRunningVerbs.Has(requestInfo.Verb) {
+			return true
+		}
+		if requestInfo.IsResourceRequest && longRunningSubresources.Has(requestInfo.Subresource) {
+			return true
+		}
+		if !requestInfo.IsResourceRequest && strings.HasPrefix(requestInfo.Path, "/debug/pprof/") {
+			return true
+		}
+		return false
+	}
+}

vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+
+	"k8s.io/klog/v2"
+)
+
+const (
+	// Constant for the retry-after interval on rate limiting.
+	// TODO: maybe make this dynamic? or user-adjustable?
+	retryAfter = "1"
+
+	// How often inflight usage metric should be updated. Because
+	// the metrics tracks maximal value over period making this
+	// longer will increase the metric value.
+	inflightUsageMetricUpdatePeriod = time.Second
+)
+
+var (
+	nonMutatingRequestVerbs = sets.NewString("get", "list", "watch")
+	watchVerbs              = sets.NewString("watch")
+)
+
+func handleError(w http.ResponseWriter, r *http.Request, err error) {
+	errorMsg := fmt.Sprintf("Internal Server Error: %#v", r.RequestURI)
+	http.Error(w, errorMsg, http.StatusInternalServerError)
+	klog.Errorf(err.Error())
+}
+
+// requestWatermark is used to track maximal numbers of requests in a particular phase of handling
+type requestWatermark struct {
+	phase                                string
+	readOnlyObserver, mutatingObserver   fcmetrics.RatioedGauge
+	lock                                 sync.Mutex
+	readOnlyWatermark, mutatingWatermark int
+}
+
+func (w *requestWatermark) recordMutating(mutatingVal int) {
+	w.mutatingObserver.Set(float64(mutatingVal))
+
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.mutatingWatermark < mutatingVal {
+		w.mutatingWatermark = mutatingVal
+	}
+}
+
+func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
+	w.readOnlyObserver.Set(float64(readOnlyVal))
+
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	if w.readOnlyWatermark < readOnlyVal {
+		w.readOnlyWatermark = readOnlyVal
+	}
+}
+
+// watermark tracks requests being executed (not waiting in a queue)
+var watermark = &requestWatermark{
+	phase: metrics.ExecutingPhase,
+}
+
+// startWatermarkMaintenance starts the goroutines to observe and maintain the specified watermark.
+func startWatermarkMaintenance(watermark *requestWatermark, stopCh <-chan struct{}) {
+	// Periodically update the inflight usage metric.
+	go wait.Until(func() {
+		watermark.lock.Lock()
+		readOnlyWatermark := watermark.readOnlyWatermark
+		mutatingWatermark := watermark.mutatingWatermark
+		watermark.readOnlyWatermark = 0
+		watermark.mutatingWatermark = 0
+		watermark.lock.Unlock()
+
+		metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
+	}, inflightUsageMetricUpdatePeriod, stopCh)
+}
+
+var initMaxInFlightOnce sync.Once
+
+func initMaxInFlight(nonMutatingLimit, mutatingLimit int) {
+	initMaxInFlightOnce.Do(func() {
+		// Fetching these gauges is delayed until after their underlying metric has been registered
+		// so that this latches onto the efficient implementation.
+		watermark.readOnlyObserver = fcmetrics.GetExecutingReadonlyConcurrency()
+		watermark.mutatingObserver = fcmetrics.GetExecutingMutatingConcurrency()
+		if nonMutatingLimit != 0 {
+			watermark.readOnlyObserver.SetDenominator(float64(nonMutatingLimit))
+			klog.V(2).InfoS("Set denominator for readonly requests", "limit", nonMutatingLimit)
+		}
+		if mutatingLimit != 0 {
+			watermark.mutatingObserver.SetDenominator(float64(mutatingLimit))
+			klog.V(2).InfoS("Set denominator for mutating requests", "limit", mutatingLimit)
+		}
+	})
+}
+
+// WithMaxInFlightLimit limits the number of in-flight requests to buffer size of the passed in channel.
+func WithMaxInFlightLimit(
+	handler http.Handler,
+	nonMutatingLimit int,
+	mutatingLimit int,
+	longRunningRequestCheck apirequest.LongRunningRequestCheck,
+) http.Handler {
+	if nonMutatingLimit == 0 && mutatingLimit == 0 {
+		return handler
+	}
+	var nonMutatingChan chan bool
+	var mutatingChan chan bool
+	if nonMutatingLimit != 0 {
+		nonMutatingChan = make(chan bool, nonMutatingLimit)
+		klog.V(2).InfoS("Initialized nonMutatingChan", "len", nonMutatingLimit)
+	} else {
+		klog.V(2).InfoS("Running with nil nonMutatingChan")
+	}
+	if mutatingLimit != 0 {
+		mutatingChan = make(chan bool, mutatingLimit)
+		klog.V(2).InfoS("Initialized mutatingChan", "len", mutatingLimit)
+	} else {
+		klog.V(2).InfoS("Running with nil mutatingChan")
+	}
+	initMaxInFlight(nonMutatingLimit, mutatingLimit)
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			handleError(w, r, fmt.Errorf("no RequestInfo found in context, handler chain must be wrong"))
+			return
+		}
+
+		// Skip tracking long running events.
+		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) {
+			handler.ServeHTTP(w, r)
+			return
+		}
+
+		var c chan bool
+		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
+		if isMutatingRequest {
+			c = mutatingChan
+		} else {
+			c = nonMutatingChan
+		}
+
+		if c == nil {
+			handler.ServeHTTP(w, r)
+		} else {
+
+			select {
+			case c <- true:
+				// We note the concurrency level both while the
+				// request is being served and after it is done being
+				// served, because both states contribute to the
+				// sampled stats on concurrency.
+				if isMutatingRequest {
+					watermark.recordMutating(len(c))
+				} else {
+					watermark.recordReadOnly(len(c))
+				}
+				defer func() {
+					<-c
+					if isMutatingRequest {
+						watermark.recordMutating(len(c))
+					} else {
+						watermark.recordReadOnly(len(c))
+					}
+				}()
+				handler.ServeHTTP(w, r)
+
+			default:
+				// at this point we're about to return a 429, BUT not all actors should be rate limited.  A system:master is so powerful
+				// that they should always get an answer.  It's a super-admin or a loopback connection.
+				if currUser, ok := apirequest.UserFrom(ctx); ok {
+					for _, group := range currUser.GetGroups() {
+						if group == user.SystemPrivilegedGroup {
+							handler.ServeHTTP(w, r)
+							return
+						}
+					}
+				}
+				// We need to split this data between buckets used for throttling.
+				metrics.RecordDroppedRequest(r, requestInfo, metrics.APIServerComponent, isMutatingRequest)
+				metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests)
+				tooManyRequests(r, w)
+			}
+		}
+	})
+}
+
+// StartMaxInFlightWatermarkMaintenance starts the goroutines to observe and maintain watermarks for max-in-flight
+// requests.
+func StartMaxInFlightWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+}
+
+func tooManyRequests(req *http.Request, w http.ResponseWriter) {
+	// Return a 429 status indicating "Too Many Requests"
+	w.Header().Set("Retry-After", retryAfter)
+	http.Error(w, "Too many requests, please try again later.", http.StatusTooManyRequests)
+}

vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -0,0 +1,325 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	flowcontrol "k8s.io/api/flowcontrol/v1beta3"
+	apitypes "k8s.io/apimachinery/pkg/types"
+	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server/httplog"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	"k8s.io/klog/v2"
+)
+
+// PriorityAndFairnessClassification identifies the results of
+// classification for API Priority and Fairness
+type PriorityAndFairnessClassification struct {
+	FlowSchemaName    string
+	FlowSchemaUID     apitypes.UID
+	PriorityLevelName string
+	PriorityLevelUID  apitypes.UID
+}
+
+// waitingMark tracks requests waiting rather than being executed
+var waitingMark = &requestWatermark{
+	phase: epmetrics.WaitingPhase,
+}
+
+var atomicMutatingExecuting, atomicReadOnlyExecuting int32
+var atomicMutatingWaiting, atomicReadOnlyWaiting int32
+
+// newInitializationSignal is defined for testing purposes.
+var newInitializationSignal = utilflowcontrol.NewInitializationSignal
+
+func truncateLogField(s string) string {
+	const maxFieldLogLength = 64
+
+	if len(s) > maxFieldLogLength {
+		s = s[0:maxFieldLogLength]
+	}
+	return s
+}
+
+var initAPFOnce sync.Once
+
+// WithPriorityAndFairness limits the number of in-flight
+// requests in a fine-grained way.
+func WithPriorityAndFairness(
+	handler http.Handler,
+	longRunningRequestCheck apirequest.LongRunningRequestCheck,
+	fcIfc utilflowcontrol.Interface,
+	workEstimator flowcontrolrequest.WorkEstimatorFunc,
+) http.Handler {
+	if fcIfc == nil {
+		klog.Warningf("priority and fairness support not found, skipping")
+		return handler
+	}
+	initAPFOnce.Do(func() {
+		initMaxInFlight(0, 0)
+		// Fetching these gauges is delayed until after their underlying metric has been registered
+		// so that this latches onto the efficient implementation.
+		waitingMark.readOnlyObserver = fcmetrics.GetWaitingReadonlyConcurrency()
+		waitingMark.mutatingObserver = fcmetrics.GetWaitingMutatingConcurrency()
+	})
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
+			return
+		}
+		user, ok := apirequest.UserFrom(ctx)
+		if !ok {
+			handleError(w, r, fmt.Errorf("no User found in context"))
+			return
+		}
+
+		isWatchRequest := watchVerbs.Has(requestInfo.Verb)
+
+		// Skip tracking long running non-watch requests.
+		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) && !isWatchRequest {
+			klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
+			handler.ServeHTTP(w, r)
+			return
+		}
+
+		var classification *PriorityAndFairnessClassification
+		noteFn := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string) {
+			classification = &PriorityAndFairnessClassification{
+				FlowSchemaName:    fs.Name,
+				FlowSchemaUID:     fs.UID,
+				PriorityLevelName: pl.Name,
+				PriorityLevelUID:  pl.UID}
+
+			httplog.AddKeyValue(ctx, "apf_pl", truncateLogField(pl.Name))
+			httplog.AddKeyValue(ctx, "apf_fs", truncateLogField(fs.Name))
+		}
+		// estimateWork is called, if at all, after noteFn
+		estimateWork := func() flowcontrolrequest.WorkEstimate {
+			if classification == nil {
+				// workEstimator is being invoked before classification of
+				// the request has completed, we should never be here though.
+				klog.ErrorS(fmt.Errorf("workEstimator is being invoked before classification of the request has completed"),
+					"Using empty FlowSchema and PriorityLevelConfiguration name", "verb", r.Method, "URI", r.RequestURI)
+
+				return workEstimator(r, "", "")
+			}
+
+			workEstimate := workEstimator(r, classification.FlowSchemaName, classification.PriorityLevelName)
+
+			fcmetrics.ObserveWorkEstimatedSeats(classification.PriorityLevelName, classification.FlowSchemaName, workEstimate.MaxSeats())
+			httplog.AddKeyValue(ctx, "apf_iseats", workEstimate.InitialSeats)
+			httplog.AddKeyValue(ctx, "apf_fseats", workEstimate.FinalSeats)
+			httplog.AddKeyValue(ctx, "apf_additionalLatency", workEstimate.AdditionalLatency)
+
+			return workEstimate
+		}
+
+		var served bool
+		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
+		noteExecutingDelta := func(delta int32) {
+			if isMutatingRequest {
+				watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
+			} else {
+				watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
+			}
+		}
+		noteWaitingDelta := func(delta int32) {
+			if isMutatingRequest {
+				waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
+			} else {
+				waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
+			}
+		}
+		queueNote := func(inQueue bool) {
+			if inQueue {
+				noteWaitingDelta(1)
+			} else {
+				noteWaitingDelta(-1)
+			}
+		}
+
+		digest := utilflowcontrol.RequestDigest{
+			RequestInfo: requestInfo,
+			User:        user,
+		}
+
+		if isWatchRequest {
+			// This channel blocks calling handler.ServeHTTP() until closed, and is closed inside execute().
+			// If APF rejects the request, it is never closed.
+			shouldStartWatchCh := make(chan struct{})
+
+			watchInitializationSignal := newInitializationSignal()
+			// This wraps the request passed to handler.ServeHTTP(),
+			// setting a context that plumbs watchInitializationSignal to storage
+			var watchReq *http.Request
+			// This is set inside execute(), prior to closing shouldStartWatchCh.
+			// If the request is rejected by APF it is left nil.
+			var forgetWatch utilflowcontrol.ForgetWatchFunc
+
+			defer func() {
+				// Protect from the situation when request will not reach storage layer
+				// and the initialization signal will not be send.
+				if watchInitializationSignal != nil {
+					watchInitializationSignal.Signal()
+				}
+				// Forget the watcher if it was registered.
+				//
+				// // This is race-free because by this point, one of the following occurred:
+				// case <-shouldStartWatchCh: execute() completed the assignment to forgetWatch
+				// case <-resultCh: Handle() completed, and Handle() does not return
+				//   while execute() is running
+				if forgetWatch != nil {
+					forgetWatch()
+				}
+			}()
+
+			execute := func() {
+				startedAt := time.Now()
+				defer func() {
+					httplog.AddKeyValue(ctx, "apf_init_latency", time.Since(startedAt))
+				}()
+				noteExecutingDelta(1)
+				defer noteExecutingDelta(-1)
+				served = true
+				setResponseHeaders(classification, w)
+
+				forgetWatch = fcIfc.RegisterWatch(r)
+
+				// Notify the main thread that we're ready to start the watch.
+				close(shouldStartWatchCh)
+
+				// Wait until the request is finished from the APF point of view
+				// (which is when its initialization is done).
+				watchInitializationSignal.Wait()
+			}
+
+			// Ensure that an item can be put to resultCh asynchronously.
+			resultCh := make(chan interface{}, 1)
+
+			// Call Handle in a separate goroutine.
+			// The reason for it is that from APF point of view, the request processing
+			// finishes as soon as watch is initialized (which is generally orders of
+			// magnitude faster then the watch request itself). This means that Handle()
+			// call finishes much faster and for performance reasons we want to reduce
+			// the number of running goroutines - so we run the shorter thing in a
+			// dedicated goroutine and the actual watch handler in the main one.
+			go func() {
+				defer func() {
+					err := recover()
+					// do not wrap the sentinel ErrAbortHandler panic value
+					if err != nil && err != http.ErrAbortHandler {
+						// Same as stdlib http server code. Manually allocate stack
+						// trace buffer size to prevent excessively large logs
+						const size = 64 << 10
+						buf := make([]byte, size)
+						buf = buf[:runtime.Stack(buf, false)]
+						err = fmt.Sprintf("%v\n%s", err, buf)
+					}
+
+					// Ensure that the result is put into resultCh independently of the panic.
+					resultCh <- err
+				}()
+
+				// We create handleCtx with explicit cancelation function.
+				// The reason for it is that Handle() underneath may start additional goroutine
+				// that is blocked on context cancellation. However, from APF point of view,
+				// we don't want to wait until the whole watch request is processed (which is
+				// when it context is actually cancelled) - we want to unblock the goroutine as
+				// soon as the request is processed from the APF point of view.
+				//
+				// Note that we explicitly do NOT call the actuall handler using that context
+				// to avoid cancelling request too early.
+				handleCtx, handleCtxCancel := context.WithCancel(ctx)
+				defer handleCtxCancel()
+
+				// Note that Handle will return irrespective of whether the request
+				// executes or is rejected. In the latter case, the function will return
+				// without calling the passed `execute` function.
+				fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
+			}()
+
+			select {
+			case <-shouldStartWatchCh:
+				watchCtx := utilflowcontrol.WithInitializationSignal(ctx, watchInitializationSignal)
+				watchReq = r.WithContext(watchCtx)
+				handler.ServeHTTP(w, watchReq)
+				// Protect from the situation when request will not reach storage layer
+				// and the initialization signal will not be send.
+				// It has to happen before waiting on the resultCh below.
+				watchInitializationSignal.Signal()
+				// TODO: Consider finishing the request as soon as Handle call panics.
+				if err := <-resultCh; err != nil {
+					panic(err)
+				}
+			case err := <-resultCh:
+				if err != nil {
+					panic(err)
+				}
+			}
+		} else {
+			execute := func() {
+				noteExecutingDelta(1)
+				defer noteExecutingDelta(-1)
+				served = true
+				setResponseHeaders(classification, w)
+
+				handler.ServeHTTP(w, r)
+			}
+
+			fcIfc.Handle(ctx, digest, noteFn, estimateWork, queueNote, execute)
+		}
+
+		if !served {
+			setResponseHeaders(classification, w)
+
+			epmetrics.RecordDroppedRequest(r, requestInfo, epmetrics.APIServerComponent, isMutatingRequest)
+			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
+			tooManyRequests(r, w)
+		}
+	})
+}
+
+// StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
+// priority-and-fairness requests.
+func StartPriorityAndFairnessWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+	startWatermarkMaintenance(waitingMark, stopCh)
+}
+
+func setResponseHeaders(classification *PriorityAndFairnessClassification, w http.ResponseWriter) {
+	if classification == nil {
+		return
+	}
+
+	// We intentionally set the UID of the flow-schema and priority-level instead of name. This is so that
+	// the names that cluster-admins choose for categorization and priority levels are not exposed, also
+	// the names might make it obvious to the users that they are rejected due to classification with low priority.
+	w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
+	w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
+}

vendor/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -0,0 +1,310 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"runtime"
+	"sync"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/server/httplog"
+)
+
+// WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
+func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck) http.Handler {
+	if longRunning == nil {
+		return handler
+	}
+	timeoutFunc := func(req *http.Request) (*http.Request, bool, func(), *apierrors.StatusError) {
+		// TODO unify this with apiserver.MaxInFlightLimit
+		ctx := req.Context()
+
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			// if this happens, the handler chain isn't setup correctly because there is no request info
+			return req, false, func() {}, apierrors.NewInternalError(fmt.Errorf("no request info found for request during timeout"))
+		}
+
+		if longRunning(req, requestInfo) {
+			return req, true, nil, nil
+		}
+
+		postTimeoutFn := func() {
+			metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
+		}
+		return req, false, postTimeoutFn, apierrors.NewTimeoutError("request did not complete within the allotted timeout", 0)
+	}
+	return WithTimeout(handler, timeoutFunc)
+}
+
+type timeoutFunc = func(*http.Request) (req *http.Request, longRunning bool, postTimeoutFunc func(), err *apierrors.StatusError)
+
+// WithTimeout returns an http.Handler that runs h with a timeout
+// determined by timeoutFunc. The new http.Handler calls h.ServeHTTP to handle
+// each request, but if a call runs for longer than its time limit, the
+// handler responds with a 504 Gateway Timeout error and the message
+// provided. (If msg is empty, a suitable default message will be sent.) After
+// the handler times out, writes by h to its http.ResponseWriter will return
+// http.ErrHandlerTimeout. If timeoutFunc returns a nil timeout channel, no
+// timeout will be enforced. recordFn is a function that will be invoked whenever
+// a timeout happens.
+func WithTimeout(h http.Handler, timeoutFunc timeoutFunc) http.Handler {
+	return &timeoutHandler{h, timeoutFunc}
+}
+
+type timeoutHandler struct {
+	handler http.Handler
+	timeout timeoutFunc
+}
+
+func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	r, longRunning, postTimeoutFn, err := t.timeout(r)
+	if longRunning {
+		t.handler.ServeHTTP(w, r)
+		return
+	}
+
+	timeoutCh := r.Context().Done()
+
+	// resultCh is used as both errCh and stopCh
+	resultCh := make(chan interface{})
+	var tw timeoutWriter
+	tw, w = newTimeoutWriter(w)
+
+	// Make a copy of request and work on it in new goroutine
+	// to avoid race condition when accessing/modifying request (e.g. headers)
+	rCopy := r.Clone(r.Context())
+	go func() {
+		defer func() {
+			err := recover()
+			// do not wrap the sentinel ErrAbortHandler panic value
+			if err != nil && err != http.ErrAbortHandler {
+				// Same as stdlib http server code. Manually allocate stack
+				// trace buffer size to prevent excessively large logs
+				const size = 64 << 10
+				buf := make([]byte, size)
+				buf = buf[:runtime.Stack(buf, false)]
+				err = fmt.Sprintf("%v\n%s", err, buf)
+			}
+			resultCh <- err
+		}()
+		t.handler.ServeHTTP(w, rCopy)
+	}()
+	select {
+	case err := <-resultCh:
+		// panic if error occurs; stop otherwise
+		if err != nil {
+			panic(err)
+		}
+		return
+	case <-timeoutCh:
+		defer func() {
+			// resultCh needs to have a reader, since the function doing
+			// the work needs to send to it. This is defer'd to ensure it runs
+			// ever if the post timeout work itself panics.
+			go func() {
+				timedOutAt := time.Now()
+				res := <-resultCh
+
+				status := metrics.PostTimeoutHandlerOK
+				if res != nil {
+					// a non nil res indicates that there was a panic.
+					status = metrics.PostTimeoutHandlerPanic
+				}
+
+				metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status)
+				err := fmt.Errorf("post-timeout activity - time-elapsed: %s, %v %q result: %v",
+					time.Since(timedOutAt), r.Method, r.URL.Path, res)
+				utilruntime.HandleError(err)
+			}()
+		}()
+		httplog.SetStacktracePredicate(r.Context(), func(status int) bool {
+			return false
+		})
+		defer postTimeoutFn()
+		tw.timeout(err)
+	}
+}
+
+type timeoutWriter interface {
+	http.ResponseWriter
+	timeout(*apierrors.StatusError)
+}
+
+func newTimeoutWriter(w http.ResponseWriter) (timeoutWriter, http.ResponseWriter) {
+	base := &baseTimeoutWriter{w: w, handlerHeaders: w.Header().Clone()}
+	wrapped := responsewriter.WrapForHTTP1Or2(base)
+
+	return base, wrapped
+}
+
+var _ http.ResponseWriter = &baseTimeoutWriter{}
+var _ responsewriter.UserProvidedDecorator = &baseTimeoutWriter{}
+
+type baseTimeoutWriter struct {
+	w http.ResponseWriter
+
+	// headers written by the normal handler
+	handlerHeaders http.Header
+
+	mu sync.Mutex
+	// if the timeout handler has timeout
+	timedOut bool
+	// if this timeout writer has wrote header
+	wroteHeader bool
+	// if this timeout writer has been hijacked
+	hijacked bool
+}
+
+func (tw *baseTimeoutWriter) Unwrap() http.ResponseWriter {
+	return tw.w
+}
+
+func (tw *baseTimeoutWriter) Header() http.Header {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return http.Header{}
+	}
+
+	return tw.handlerHeaders
+}
+
+func (tw *baseTimeoutWriter) Write(p []byte) (int, error) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return 0, http.ErrHandlerTimeout
+	}
+	if tw.hijacked {
+		return 0, http.ErrHijacked
+	}
+
+	if !tw.wroteHeader {
+		copyHeaders(tw.w.Header(), tw.handlerHeaders)
+		tw.wroteHeader = true
+	}
+	return tw.w.Write(p)
+}
+
+func (tw *baseTimeoutWriter) Flush() {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Flusher if the inner object (tw.w) implements http.Flusher.
+	tw.w.(http.Flusher).Flush()
+}
+
+func (tw *baseTimeoutWriter) WriteHeader(code int) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut || tw.wroteHeader || tw.hijacked {
+		return
+	}
+
+	copyHeaders(tw.w.Header(), tw.handlerHeaders)
+	tw.wroteHeader = true
+	tw.w.WriteHeader(code)
+}
+
+func copyHeaders(dst, src http.Header) {
+	for k, v := range src {
+		dst[k] = v
+	}
+}
+
+func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	tw.timedOut = true
+
+	// The timeout writer has not been used by the inner handler.
+	// We can safely timeout the HTTP request by sending by a timeout
+	// handler
+	if !tw.wroteHeader && !tw.hijacked {
+		tw.w.WriteHeader(http.StatusGatewayTimeout)
+		enc := json.NewEncoder(tw.w)
+		enc.Encode(&err.ErrStatus)
+	} else {
+		// The timeout writer has been used by the inner handler. There is
+		// no way to timeout the HTTP request at the point. We have to shutdown
+		// the connection for HTTP1 or reset stream for HTTP2.
+		//
+		// Note from the golang's docs:
+		// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+		// that the effect of the panic was isolated to the active request.
+		// It recovers the panic, logs a stack trace to the server error log,
+		// and either closes the network connection or sends an HTTP/2
+		// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+		// the client sees an interrupted response but the server doesn't log
+		// an error, panic with the value ErrAbortHandler.
+		//
+		// We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs
+		panic(http.ErrAbortHandler)
+	}
+}
+
+func (tw *baseTimeoutWriter) CloseNotify() <-chan bool {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		done := make(chan bool)
+		close(done)
+		return done
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.CloseNotifier if the inner object (tw.w) implements http.CloseNotifier.
+	return tw.w.(http.CloseNotifier).CloseNotify()
+}
+
+func (tw *baseTimeoutWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	tw.mu.Lock()
+	defer tw.mu.Unlock()
+
+	if tw.timedOut {
+		return nil, nil, http.ErrHandlerTimeout
+	}
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Hijacker if the inner object (tw.w) implements http.Hijacker.
+	conn, rw, err := tw.w.(http.Hijacker).Hijack()
+	if err == nil {
+		tw.hijacked = true
+	}
+	return conn, rw, err
+}

vendor/k8s.io/apiserver/pkg/server/filters/waitgroup.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/client-go/kubernetes/scheme"
+)
+
+// WithWaitGroup adds all non long-running requests to wait group, which is used for graceful shutdown.
+func WithWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg *utilwaitgroup.SafeWaitGroup) http.Handler {
+	// NOTE: both WithWaitGroup and WithRetryAfter must use the same exact isRequestExemptFunc 'isRequestExemptFromRetryAfter,
+	// otherwise SafeWaitGroup might wait indefinitely and will prevent the server from shutting down gracefully.
+	return withWaitGroup(handler, longRunning, wg, isRequestExemptFromRetryAfter)
+}
+
+func withWaitGroup(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, wg *utilwaitgroup.SafeWaitGroup, isRequestExemptFn isRequestExemptFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if !ok {
+			// if this happens, the handler chain isn't setup correctly because there is no request info
+			responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
+			return
+		}
+
+		if longRunning(req, requestInfo) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		if err := wg.Add(1); err != nil {
+			// shutdown delay duration has elapsed and SafeWaitGroup.Wait has been invoked,
+			// this means 'WithRetryAfter' has started sending Retry-After response.
+			// we are going to exempt the same set of requests that WithRetryAfter are
+			// exempting from being rejected with a Retry-After response.
+			if isRequestExemptFn(req) {
+				handler.ServeHTTP(w, req)
+				return
+			}
+
+			// When apiserver is shutting down, signal clients to retry
+			// There is a good chance the client hit a different server, so a tight retry is good for client responsiveness.
+			w.Header().Add("Retry-After", "1")
+			w.Header().Set("Content-Type", runtime.ContentTypeJSON)
+			w.Header().Set("X-Content-Type-Options", "nosniff")
+			statusErr := apierrors.NewServiceUnavailable("apiserver is shutting down").Status()
+			w.WriteHeader(int(statusErr.Code))
+			fmt.Fprintln(w, runtime.EncodeOrDie(scheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), &statusErr))
+			return
+		}
+
+		defer wg.Done()
+		handler.ServeHTTP(w, req)
+	})
+}

vendor/k8s.io/apiserver/pkg/server/filters/with_retry_after.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+	"strings"
+)
+
+var (
+	// health probes and metrics scraping are never rejected, we will continue
+	// serving these requests after shutdown delay duration elapses.
+	pathPrefixesExemptFromRetryAfter = []string{
+		"/readyz",
+		"/livez",
+		"/healthz",
+		"/metrics",
+	}
+)
+
+// isRequestExemptFunc returns true if the request should not be rejected,
+// with a Retry-After response, otherwise it returns false.
+type isRequestExemptFunc func(*http.Request) bool
+
+// retryAfterParams dictates how the Retry-After response is constructed
+type retryAfterParams struct {
+	// TearDownConnection is true when we should send a 'Connection: close'
+	// header in the response so net/http can tear down the TCP connection.
+	TearDownConnection bool
+
+	// Message describes why Retry-After response has been sent by the server
+	Message string
+}
+
+// shouldRespondWithRetryAfterFunc returns true if the requests should
+// be rejected with a Retry-After response once certain conditions are met.
+// The retryAfterParams returned contains instructions on how to
+// construct the Retry-After response.
+type shouldRespondWithRetryAfterFunc func() (*retryAfterParams, bool)
+
+// WithRetryAfter rejects any incoming new request(s) with a 429
+// if the specified shutdownDelayDurationElapsedFn channel is closed
+//
+// It includes new request(s) on a new or an existing TCP connection
+// Any new request(s) arriving after shutdownDelayDurationElapsedFn is closed
+// are replied with a 429 and the following response headers:
+//   - 'Retry-After: N` (so client can retry after N seconds, hopefully on a new apiserver instance)
+//   - 'Connection: close': tear down the TCP connection
+//
+// TODO: is there a way to merge WithWaitGroup and this filter?
+func WithRetryAfter(handler http.Handler, shutdownDelayDurationElapsedCh <-chan struct{}) http.Handler {
+	shutdownRetryAfterParams := &retryAfterParams{
+		TearDownConnection: true,
+		Message:            "The apiserver is shutting down, please try again later.",
+	}
+
+	// NOTE: both WithRetryAfter and WithWaitGroup must use the same exact isRequestExemptFunc 'isRequestExemptFromRetryAfter,
+	// otherwise SafeWaitGroup might wait indefinitely and will prevent the server from shutting down gracefully.
+	return withRetryAfter(handler, isRequestExemptFromRetryAfter, func() (*retryAfterParams, bool) {
+		select {
+		case <-shutdownDelayDurationElapsedCh:
+			return shutdownRetryAfterParams, true
+		default:
+			return nil, false
+		}
+	})
+}
+
+func withRetryAfter(handler http.Handler, isRequestExemptFn isRequestExemptFunc, shouldRespondWithRetryAfterFn shouldRespondWithRetryAfterFunc) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		params, send := shouldRespondWithRetryAfterFn()
+		if !send || isRequestExemptFn(req) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		// If we are here this means it's time to send Retry-After response
+		//
+		// Copied from net/http2 library
+		// "Connection" headers aren't allowed in HTTP/2 (RFC 7540, 8.1.2.2),
+		// but respect "Connection" == "close" to mean sending a GOAWAY and tearing
+		// down the TCP connection when idle, like we do for HTTP/1.
+		if params.TearDownConnection {
+			w.Header().Set("Connection", "close")
+		}
+
+		// Return a 429 status asking the client to try again after 5 seconds
+		w.Header().Set("Retry-After", "5")
+		http.Error(w, params.Message, http.StatusTooManyRequests)
+	})
+}
+
+// isRequestExemptFromRetryAfter returns true if the given request should be exempt
+// from being rejected with a 'Retry-After' response.
+// NOTE: both 'WithRetryAfter' and 'WithWaitGroup' filters should use this function
+// to exempt the set of requests from being rejected or tracked.
+func isRequestExemptFromRetryAfter(r *http.Request) bool {
+	return isKubeApiserverUserAgent(r) || hasExemptPathPrefix(r)
+}
+
+// isKubeApiserverUserAgent returns true if the user-agent matches
+// the one set by the local loopback.
+// NOTE: we can't look up the authenticated user informaion from the
+// request context since the authentication filter has not executed yet.
+func isKubeApiserverUserAgent(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), "kube-apiserver/")
+}
+
+func hasExemptPathPrefix(r *http.Request) bool {
+	for _, whiteListedPrefix := range pathPrefixesExemptFromRetryAfter {
+		if strings.HasPrefix(r.URL.Path, whiteListedPrefix) {
+			return true
+		}
+	}
+	return false
+}

vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"fmt"
+	"net/http"
+
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/klog/v2"
+)
+
+// WithPanicRecovery wraps an http Handler to recover and log panics (except in the special case of http.ErrAbortHandler panics, which suppress logging).
+func WithPanicRecovery(handler http.Handler, resolver request.RequestInfoResolver) http.Handler {
+	return withPanicRecovery(handler, func(w http.ResponseWriter, req *http.Request, err interface{}) {
+		if err == http.ErrAbortHandler {
+			// Honor the http.ErrAbortHandler sentinel panic value
+			//
+			// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+			// that the effect of the panic was isolated to the active request.
+			// It recovers the panic, logs a stack trace to the server error log,
+			// and either closes the network connection or sends an HTTP/2
+			// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+			// the client sees an interrupted response but the server doesn't log
+			// an error, panic with the value ErrAbortHandler.
+			//
+			// Note that HandleCrash function is actually crashing, after calling the handlers
+			if info, err := resolver.NewRequestInfo(req); err != nil {
+				metrics.RecordRequestAbort(req, nil)
+			} else {
+				metrics.RecordRequestAbort(req, info)
+			}
+			// This call can have different handlers, but the default chain rate limits. Call it after the metrics are updated
+			// in case the rate limit delays it.  If you outrun the rate for this one timed out requests, something has gone
+			// seriously wrong with your server, but generally having a logging signal for timeouts is useful.
+			runtime.HandleError(fmt.Errorf("timeout or abort while handling: method=%v URI=%q audit-ID=%q", req.Method, req.RequestURI, audit.GetAuditIDTruncated(req.Context())))
+			return
+		}
+		http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
+		klog.ErrorS(nil, "apiserver panic'd", "method", req.Method, "URI", req.RequestURI, "audit-ID", audit.GetAuditIDTruncated(req.Context()))
+	})
+}
+
+// WithHTTPLogging enables logging of incoming requests.
+func WithHTTPLogging(handler http.Handler) http.Handler {
+	return httplog.WithLogging(handler, httplog.DefaultStacktracePred)
+}
+
+func withPanicRecovery(handler http.Handler, crashHandler func(http.ResponseWriter, *http.Request, interface{})) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		defer runtime.HandleCrash(func(err interface{}) {
+			crashHandler(w, req, err)
+		})
+
+		// Dispatch to the internal handler
+		handler.ServeHTTP(w, req)
+	})
+}

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -0,0 +1,919 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	gpath "path"
+	"strings"
+	"sync"
+	"time"
+
+	systemd "github.com/coreos/go-systemd/v22/daemon"
+
+	apidiscoveryv2beta1 "k8s.io/api/apidiscovery/v2beta1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	genericapi "k8s.io/apiserver/pkg/endpoints"
+	"k8s.io/apiserver/pkg/endpoints/discovery"
+	discoveryendpoint "k8s.io/apiserver/pkg/endpoints/discovery/aggregated"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/routes"
+	"k8s.io/apiserver/pkg/storageversion"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+	openapibuilder2 "k8s.io/kube-openapi/pkg/builder"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/handler"
+	"k8s.io/kube-openapi/pkg/handler3"
+	openapiutil "k8s.io/kube-openapi/pkg/util"
+	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+	"k8s.io/utils/clock"
+)
+
+// Info about an API group.
+type APIGroupInfo struct {
+	PrioritizedVersions []schema.GroupVersion
+	// Info about the resources in this group. It's a map from version to resource to the storage.
+	VersionedResourcesStorageMap map[string]map[string]rest.Storage
+	// OptionsExternalVersion controls the APIVersion used for common objects in the
+	// schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may
+	// define a version "v1beta1" but want to use the Kubernetes "v1" internal objects.
+	// If nil, defaults to groupMeta.GroupVersion.
+	// TODO: Remove this when https://github.com/kubernetes/kubernetes/issues/19018 is fixed.
+	OptionsExternalVersion *schema.GroupVersion
+	// MetaGroupVersion defaults to "meta.k8s.io/v1" and is the scheme group version used to decode
+	// common API implementations like ListOptions. Future changes will allow this to vary by group
+	// version (for when the inevitable meta/v2 group emerges).
+	MetaGroupVersion *schema.GroupVersion
+
+	// Scheme includes all of the types used by this group and how to convert between them (or
+	// to convert objects from outside of this group that are accepted in this API).
+	// TODO: replace with interfaces
+	Scheme *runtime.Scheme
+	// NegotiatedSerializer controls how this group encodes and decodes data
+	NegotiatedSerializer runtime.NegotiatedSerializer
+	// ParameterCodec performs conversions for query parameters passed to API calls
+	ParameterCodec runtime.ParameterCodec
+
+	// StaticOpenAPISpec is the spec derived from the definitions of all resources installed together.
+	// It is set during InstallAPIGroups, InstallAPIGroup, and InstallLegacyAPIGroup.
+	StaticOpenAPISpec *spec.Swagger
+}
+
+func (a *APIGroupInfo) destroyStorage() {
+	for _, stores := range a.VersionedResourcesStorageMap {
+		for _, store := range stores {
+			store.Destroy()
+		}
+	}
+}
+
+// GenericAPIServer contains state for a Kubernetes cluster api server.
+type GenericAPIServer struct {
+	// discoveryAddresses is used to build cluster IPs for discovery.
+	discoveryAddresses discovery.Addresses
+
+	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
+	LoopbackClientConfig *restclient.Config
+
+	// minRequestTimeout is how short the request timeout can be.  This is used to build the RESTHandler
+	minRequestTimeout time.Duration
+
+	// ShutdownTimeout is the timeout used for server shutdown. This specifies the timeout before server
+	// gracefully shutdown returns.
+	ShutdownTimeout time.Duration
+
+	// legacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
+	// to InstallLegacyAPIGroup
+	legacyAPIGroupPrefixes sets.String
+
+	// admissionControl is used to build the RESTStorage that backs an API Group.
+	admissionControl admission.Interface
+
+	// SecureServingInfo holds configuration of the TLS server.
+	SecureServingInfo *SecureServingInfo
+
+	// ExternalAddress is the address (hostname or IP and port) that should be used in
+	// external (public internet) URLs for this GenericAPIServer.
+	ExternalAddress string
+
+	// Serializer controls how common API objects not in a group/version prefix are serialized for this server.
+	// Individual APIGroups may define their own serializers.
+	Serializer runtime.NegotiatedSerializer
+
+	// "Outputs"
+	// Handler holds the handlers being used by this API server
+	Handler *APIServerHandler
+
+	// listedPathProvider is a lister which provides the set of paths to show at /
+	listedPathProvider routes.ListedPathProvider
+
+	// DiscoveryGroupManager serves /apis in an unaggregated form.
+	DiscoveryGroupManager discovery.GroupManager
+
+	// AggregatedDiscoveryGroupManager serves /apis in an aggregated form.
+	AggregatedDiscoveryGroupManager discoveryendpoint.ResourceManager
+
+	// AggregatedLegacyDiscoveryGroupManager serves /api in an aggregated form.
+	AggregatedLegacyDiscoveryGroupManager discoveryendpoint.ResourceManager
+
+	// Enable swagger and/or OpenAPI if these configs are non-nil.
+	openAPIConfig *openapicommon.Config
+
+	// Enable swagger and/or OpenAPI V3 if these configs are non-nil.
+	openAPIV3Config *openapicommon.Config
+
+	// SkipOpenAPIInstallation indicates not to install the OpenAPI handler
+	// during PrepareRun.
+	// Set this to true when the specific API Server has its own OpenAPI handler
+	// (e.g. kube-aggregator)
+	skipOpenAPIInstallation bool
+
+	// OpenAPIVersionedService controls the /openapi/v2 endpoint, and can be used to update the served spec.
+	// It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true.
+	OpenAPIVersionedService *handler.OpenAPIService
+
+	// OpenAPIV3VersionedService controls the /openapi/v3 endpoint and can be used to update the served spec.
+	// It is set during PrepareRun if `openAPIConfig` is non-nil unless `skipOpenAPIInstallation` is true.
+	OpenAPIV3VersionedService *handler3.OpenAPIService
+
+	// StaticOpenAPISpec is the spec derived from the restful container endpoints.
+	// It is set during PrepareRun.
+	StaticOpenAPISpec *spec.Swagger
+
+	// PostStartHooks are each called after the server has started listening, in a separate go func for each
+	// with no guarantee of ordering between them.  The map key is a name used for error reporting.
+	// It may kill the process with a panic if it wishes to by returning an error.
+	postStartHookLock      sync.Mutex
+	postStartHooks         map[string]postStartHookEntry
+	postStartHooksCalled   bool
+	disabledPostStartHooks sets.String
+
+	preShutdownHookLock    sync.Mutex
+	preShutdownHooks       map[string]preShutdownHookEntry
+	preShutdownHooksCalled bool
+
+	// healthz checks
+	healthzLock            sync.Mutex
+	healthzChecks          []healthz.HealthChecker
+	healthzChecksInstalled bool
+	// livez checks
+	livezLock            sync.Mutex
+	livezChecks          []healthz.HealthChecker
+	livezChecksInstalled bool
+	// readyz checks
+	readyzLock            sync.Mutex
+	readyzChecks          []healthz.HealthChecker
+	readyzChecksInstalled bool
+	livezGracePeriod      time.Duration
+	livezClock            clock.Clock
+
+	// auditing. The backend is started before the server starts listening.
+	AuditBackend audit.Backend
+
+	// Authorizer determines whether a user is allowed to make a certain request. The Handler does a preliminary
+	// authorization check using the request URI but it may be necessary to make additional checks, such as in
+	// the create-on-update case
+	Authorizer authorizer.Authorizer
+
+	// EquivalentResourceRegistry provides information about resources equivalent to a given resource,
+	// and the kind associated with a given resource. As resources are installed, they are registered here.
+	EquivalentResourceRegistry runtime.EquivalentResourceRegistry
+
+	// delegationTarget is the next delegate in the chain. This is never nil.
+	delegationTarget DelegationTarget
+
+	// HandlerChainWaitGroup allows you to wait for all chain handlers finish after the server shutdown.
+	HandlerChainWaitGroup *utilwaitgroup.SafeWaitGroup
+
+	// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
+	// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
+	// but /readyz will return failure.
+	ShutdownDelayDuration time.Duration
+
+	// The limit on the request body size that would be accepted and decoded in a write request.
+	// 0 means no limit.
+	maxRequestBodyBytes int64
+
+	// APIServerID is the ID of this API server
+	APIServerID string
+
+	// StorageVersionManager holds the storage versions of the API resources installed by this server.
+	StorageVersionManager storageversion.Manager
+
+	// Version will enable the /version endpoint if non-nil
+	Version *version.Info
+
+	// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
+	lifecycleSignals lifecycleSignals
+
+	// destroyFns contains a list of functions that should be called on shutdown to clean up resources.
+	destroyFns []func()
+
+	// muxAndDiscoveryCompleteSignals holds signals that indicate all known HTTP paths have been registered.
+	// it exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler.
+	// it is exposed for easier composition of the individual servers.
+	// the primary users of this field are the WithMuxCompleteProtection filter and the NotFoundHandler
+	muxAndDiscoveryCompleteSignals map[string]<-chan struct{}
+
+	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
+	// Server during the graceful termination of the apiserver. If true, we wait
+	// for non longrunning requests in flight to be drained and then initiate a
+	// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
+	// Server as soon as ShutdownDelayDuration has elapsed.
+	// If enabled, after ShutdownDelayDuration elapses, any incoming request is
+	// rejected with a 429 status code and a 'Retry-After' response.
+	ShutdownSendRetryAfter bool
+}
+
+// DelegationTarget is an interface which allows for composition of API servers with top level handling that works
+// as expected.
+type DelegationTarget interface {
+	// UnprotectedHandler returns a handler that is NOT protected by a normal chain
+	UnprotectedHandler() http.Handler
+
+	// PostStartHooks returns the post-start hooks that need to be combined
+	PostStartHooks() map[string]postStartHookEntry
+
+	// PreShutdownHooks returns the pre-stop hooks that need to be combined
+	PreShutdownHooks() map[string]preShutdownHookEntry
+
+	// HealthzChecks returns the healthz checks that need to be combined
+	HealthzChecks() []healthz.HealthChecker
+
+	// ListedPaths returns the paths for supporting an index
+	ListedPaths() []string
+
+	// NextDelegate returns the next delegationTarget in the chain of delegations
+	NextDelegate() DelegationTarget
+
+	// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
+	PrepareRun() preparedGenericAPIServer
+
+	// MuxAndDiscoveryCompleteSignals exposes registered signals that indicate if all known HTTP paths have been installed.
+	MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{}
+
+	// Destroy cleans up its resources on shutdown.
+	// Destroy has to be implemented in thread-safe way and be prepared
+	// for being called more than once.
+	Destroy()
+}
+
+func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
+	// when we delegate, we need the server we're delegating to choose whether or not to use gorestful
+	return s.Handler.Director
+}
+func (s *GenericAPIServer) PostStartHooks() map[string]postStartHookEntry {
+	return s.postStartHooks
+}
+func (s *GenericAPIServer) PreShutdownHooks() map[string]preShutdownHookEntry {
+	return s.preShutdownHooks
+}
+func (s *GenericAPIServer) HealthzChecks() []healthz.HealthChecker {
+	return s.healthzChecks
+}
+func (s *GenericAPIServer) ListedPaths() []string {
+	return s.listedPathProvider.ListedPaths()
+}
+
+func (s *GenericAPIServer) NextDelegate() DelegationTarget {
+	return s.delegationTarget
+}
+
+// RegisterMuxAndDiscoveryCompleteSignal registers the given signal that will be used to determine if all known
+// HTTP paths have been registered. It is okay to call this method after instantiating the generic server but before running.
+func (s *GenericAPIServer) RegisterMuxAndDiscoveryCompleteSignal(signalName string, signal <-chan struct{}) error {
+	if _, exists := s.muxAndDiscoveryCompleteSignals[signalName]; exists {
+		return fmt.Errorf("%s already registered", signalName)
+	}
+	s.muxAndDiscoveryCompleteSignals[signalName] = signal
+	return nil
+}
+
+func (s *GenericAPIServer) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} {
+	return s.muxAndDiscoveryCompleteSignals
+}
+
+// RegisterDestroyFunc registers a function that will be called during Destroy().
+// The function have to be idempotent and prepared to be called more than once.
+func (s *GenericAPIServer) RegisterDestroyFunc(destroyFn func()) {
+	s.destroyFns = append(s.destroyFns, destroyFn)
+}
+
+// Destroy cleans up all its and its delegation target resources on shutdown.
+// It starts with destroying its own resources and later proceeds with
+// its delegation target.
+func (s *GenericAPIServer) Destroy() {
+	for _, destroyFn := range s.destroyFns {
+		destroyFn()
+	}
+	if s.delegationTarget != nil {
+		s.delegationTarget.Destroy()
+	}
+}
+
+type emptyDelegate struct {
+	// handler is called at the end of the delegation chain
+	// when a request has been made against an unregistered HTTP path the individual servers will simply pass it through until it reaches the handler.
+	handler http.Handler
+}
+
+func NewEmptyDelegate() DelegationTarget {
+	return emptyDelegate{}
+}
+
+// NewEmptyDelegateWithCustomHandler allows for registering a custom handler usually for special handling of 404 requests
+func NewEmptyDelegateWithCustomHandler(handler http.Handler) DelegationTarget {
+	return emptyDelegate{handler}
+}
+
+func (s emptyDelegate) UnprotectedHandler() http.Handler {
+	return s.handler
+}
+func (s emptyDelegate) PostStartHooks() map[string]postStartHookEntry {
+	return map[string]postStartHookEntry{}
+}
+func (s emptyDelegate) PreShutdownHooks() map[string]preShutdownHookEntry {
+	return map[string]preShutdownHookEntry{}
+}
+func (s emptyDelegate) HealthzChecks() []healthz.HealthChecker {
+	return []healthz.HealthChecker{}
+}
+func (s emptyDelegate) ListedPaths() []string {
+	return []string{}
+}
+func (s emptyDelegate) NextDelegate() DelegationTarget {
+	return nil
+}
+func (s emptyDelegate) PrepareRun() preparedGenericAPIServer {
+	return preparedGenericAPIServer{nil}
+}
+func (s emptyDelegate) MuxAndDiscoveryCompleteSignals() map[string]<-chan struct{} {
+	return map[string]<-chan struct{}{}
+}
+func (s emptyDelegate) Destroy() {
+}
+
+// preparedGenericAPIServer is a private wrapper that enforces a call of PrepareRun() before Run can be invoked.
+type preparedGenericAPIServer struct {
+	*GenericAPIServer
+}
+
+// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
+func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
+	s.delegationTarget.PrepareRun()
+
+	if s.openAPIConfig != nil && !s.skipOpenAPIInstallation {
+		s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{
+			Config: s.openAPIConfig,
+		}.InstallV2(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
+	}
+
+	if s.openAPIV3Config != nil && !s.skipOpenAPIInstallation {
+		if utilfeature.DefaultFeatureGate.Enabled(features.OpenAPIV3) {
+			s.OpenAPIV3VersionedService = routes.OpenAPI{
+				Config: s.openAPIV3Config,
+			}.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
+		}
+	}
+
+	s.installHealthz()
+	s.installLivez()
+
+	// as soon as shutdown is initiated, readiness should start failing
+	readinessStopCh := s.lifecycleSignals.ShutdownInitiated.Signaled()
+	err := s.addReadyzShutdownCheck(readinessStopCh)
+	if err != nil {
+		klog.Errorf("Failed to install readyz shutdown check %s", err)
+	}
+	s.installReadyz()
+
+	return preparedGenericAPIServer{s}
+}
+
+// Run spawns the secure http server. It only returns if stopCh is closed
+// or the secure port cannot be listened on initially.
+// This is the diagram of what channels/signals are dependent on each other:
+//
+// |                                  stopCh
+// |                                    |
+// |           ---------------------------------------------------------
+// |           |                                                       |
+// |    ShutdownInitiated (shutdownInitiatedCh)                        |
+// |           |                                                       |
+// | (ShutdownDelayDuration)                                    (PreShutdownHooks)
+// |           |                                                       |
+// |  AfterShutdownDelayDuration (delayedStopCh)   PreShutdownHooksStopped (preShutdownHooksHasStoppedCh)
+// |           |                                                       |
+// |           |-------------------------------------------------------|
+// |                                    |
+// |                                    |
+// |               NotAcceptingNewRequest (notAcceptingNewRequestCh)
+// |                                    |
+// |                                    |
+// |           |---------------------------------------------------------|
+// |           |                        |              |                 |
+// |        [without                 [with             |                 |
+// | ShutdownSendRetryAfter]  ShutdownSendRetryAfter]  |                 |
+// |           |                        |              |                 |
+// |           |                        ---------------|                 |
+// |           |                                       |                 |
+// |           |                         (HandlerChainWaitGroup::Wait)   |
+// |           |                                       |                 |
+// |           |                    InFlightRequestsDrained (drainedCh)  |
+// |           |                                       |                 |
+// |           ----------------------------------------|-----------------|
+// |                                 |                 |
+// |                       stopHttpServerCh     (AuditBackend::Shutdown())
+// |                                 |
+// |                       listenerStoppedCh
+// |                                 |
+// |      HTTPServerStoppedListening (httpServerStoppedListeningCh)
+func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
+	delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration
+	shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated
+
+	// Clean up resources on shutdown.
+	defer s.Destroy()
+
+	// spawn a new goroutine for closing the MuxAndDiscoveryComplete signal
+	// registration happens during construction of the generic api server
+	// the last server in the chain aggregates signals from the previous instances
+	go func() {
+		for _, muxAndDiscoveryCompletedSignal := range s.GenericAPIServer.MuxAndDiscoveryCompleteSignals() {
+			select {
+			case <-muxAndDiscoveryCompletedSignal:
+				continue
+			case <-stopCh:
+				klog.V(1).Infof("haven't completed %s, stop requested", s.lifecycleSignals.MuxAndDiscoveryComplete.Name())
+				return
+			}
+		}
+		s.lifecycleSignals.MuxAndDiscoveryComplete.Signal()
+		klog.V(1).Infof("%s has all endpoints registered and discovery information is complete", s.lifecycleSignals.MuxAndDiscoveryComplete.Name())
+	}()
+
+	go func() {
+		defer delayedStopCh.Signal()
+		defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", delayedStopCh.Name())
+
+		<-stopCh
+
+		// As soon as shutdown is initiated, /readyz should start returning failure.
+		// This gives the load balancer a window defined by ShutdownDelayDuration to detect that /readyz is red
+		// and stop sending traffic to this server.
+		shutdownInitiatedCh.Signal()
+		klog.V(1).InfoS("[graceful-termination] shutdown event", "name", shutdownInitiatedCh.Name())
+
+		time.Sleep(s.ShutdownDelayDuration)
+	}()
+
+	// close socket after delayed stopCh
+	shutdownTimeout := s.ShutdownTimeout
+	if s.ShutdownSendRetryAfter {
+		// when this mode is enabled, we do the following:
+		// - the server will continue to listen until all existing requests in flight
+		//   (not including active long running requests) have been drained.
+		// - once drained, http Server Shutdown is invoked with a timeout of 2s,
+		//   net/http waits for 1s for the peer to respond to a GO_AWAY frame, so
+		//   we should wait for a minimum of 2s
+		shutdownTimeout = 2 * time.Second
+		klog.V(1).InfoS("[graceful-termination] using HTTP Server shutdown timeout", "ShutdownTimeout", shutdownTimeout)
+	}
+
+	notAcceptingNewRequestCh := s.lifecycleSignals.NotAcceptingNewRequest
+	drainedCh := s.lifecycleSignals.InFlightRequestsDrained
+	stopHttpServerCh := make(chan struct{})
+	go func() {
+		defer close(stopHttpServerCh)
+
+		timeToStopHttpServerCh := notAcceptingNewRequestCh.Signaled()
+		if s.ShutdownSendRetryAfter {
+			timeToStopHttpServerCh = drainedCh.Signaled()
+		}
+
+		<-timeToStopHttpServerCh
+	}()
+
+	// Start the audit backend before any request comes in. This means we must call Backend.Run
+	// before http server start serving. Otherwise the Backend.ProcessEvents call might block.
+	// AuditBackend.Run will stop as soon as all in-flight requests are drained.
+	if s.AuditBackend != nil {
+		if err := s.AuditBackend.Run(drainedCh.Signaled()); err != nil {
+			return fmt.Errorf("failed to run the audit backend: %v", err)
+		}
+	}
+
+	stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(stopHttpServerCh, shutdownTimeout)
+	if err != nil {
+		return err
+	}
+
+	httpServerStoppedListeningCh := s.lifecycleSignals.HTTPServerStoppedListening
+	go func() {
+		<-listenerStoppedCh
+		httpServerStoppedListeningCh.Signal()
+		klog.V(1).InfoS("[graceful-termination] shutdown event", "name", httpServerStoppedListeningCh.Name())
+	}()
+
+	// we don't accept new request as soon as both ShutdownDelayDuration has
+	// elapsed and preshutdown hooks have completed.
+	preShutdownHooksHasStoppedCh := s.lifecycleSignals.PreShutdownHooksStopped
+	go func() {
+		defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", notAcceptingNewRequestCh.Name())
+		defer notAcceptingNewRequestCh.Signal()
+
+		// wait for the delayed stopCh before closing the handler chain
+		<-delayedStopCh.Signaled()
+
+		// Additionally wait for preshutdown hooks to also be finished, as some of them need
+		// to send API calls to clean up after themselves (e.g. lease reconcilers removing
+		// itself from the active servers).
+		<-preShutdownHooksHasStoppedCh.Signaled()
+	}()
+
+	go func() {
+		defer klog.V(1).InfoS("[graceful-termination] shutdown event", "name", drainedCh.Name())
+		defer drainedCh.Signal()
+
+		// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
+		<-notAcceptingNewRequestCh.Signaled()
+
+		// Wait for all requests to finish, which are bounded by the RequestTimeout variable.
+		// once HandlerChainWaitGroup.Wait is invoked, the apiserver is
+		// expected to reject any incoming request with a {503, Retry-After}
+		// response via the WithWaitGroup filter. On the contrary, we observe
+		// that incoming request(s) get a 'connection refused' error, this is
+		// because, at this point, we have called 'Server.Shutdown' and
+		// net/http server has stopped listening. This causes incoming
+		// request to get a 'connection refused' error.
+		// On the other hand, if 'ShutdownSendRetryAfter' is enabled incoming
+		// requests will be rejected with a {429, Retry-After} since
+		// 'Server.Shutdown' will be invoked only after in-flight requests
+		// have been drained.
+		// TODO: can we consolidate these two modes of graceful termination?
+		s.HandlerChainWaitGroup.Wait()
+	}()
+
+	klog.V(1).Info("[graceful-termination] waiting for shutdown to be initiated")
+	<-stopCh
+
+	// run shutdown hooks directly. This includes deregistering from
+	// the kubernetes endpoint in case of kube-apiserver.
+	func() {
+		defer func() {
+			preShutdownHooksHasStoppedCh.Signal()
+			klog.V(1).InfoS("[graceful-termination] pre-shutdown hooks completed", "name", preShutdownHooksHasStoppedCh.Name())
+		}()
+		err = s.RunPreShutdownHooks()
+	}()
+	if err != nil {
+		return err
+	}
+
+	// Wait for all requests in flight to drain, bounded by the RequestTimeout variable.
+	<-drainedCh.Signaled()
+
+	if s.AuditBackend != nil {
+		s.AuditBackend.Shutdown()
+		klog.V(1).InfoS("[graceful-termination] audit backend shutdown completed")
+	}
+
+	// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
+	<-listenerStoppedCh
+	<-stoppedCh
+
+	klog.V(1).Info("[graceful-termination] apiserver is exiting")
+	return nil
+}
+
+// NonBlockingRun spawns the secure http server. An error is
+// returned if the secure port cannot be listened on.
+// The returned channel is closed when the (asynchronous) termination is finished.
+func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
+	// Use an internal stop channel to allow cleanup of the listeners on error.
+	internalStopCh := make(chan struct{})
+	var stoppedCh <-chan struct{}
+	var listenerStoppedCh <-chan struct{}
+	if s.SecureServingInfo != nil && s.Handler != nil {
+		var err error
+		stoppedCh, listenerStoppedCh, err = s.SecureServingInfo.Serve(s.Handler, shutdownTimeout, internalStopCh)
+		if err != nil {
+			close(internalStopCh)
+			return nil, nil, err
+		}
+	}
+
+	// Now that listener have bound successfully, it is the
+	// responsibility of the caller to close the provided channel to
+	// ensure cleanup.
+	go func() {
+		<-stopCh
+		close(internalStopCh)
+	}()
+
+	s.RunPostStartHooks(stopCh)
+
+	if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil {
+		klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
+	}
+
+	return stoppedCh, listenerStoppedCh, nil
+}
+
+// installAPIResources is a private method for installing the REST storage backing each api groupversionresource
+func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, openAPIModels openapiproto.Models) error {
+	var resourceInfos []*storageversion.ResourceInfo
+	for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
+		if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
+			klog.Warningf("Skipping API %v because it has no resources.", groupVersion)
+			continue
+		}
+
+		apiGroupVersion, err := s.getAPIGroupVersion(apiGroupInfo, groupVersion, apiPrefix)
+		if err != nil {
+			return err
+		}
+		if apiGroupInfo.OptionsExternalVersion != nil {
+			apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
+		}
+		apiGroupVersion.OpenAPIModels = openAPIModels
+
+		if openAPIModels != nil {
+			typeConverter, err := fieldmanager.NewTypeConverter(openAPIModels, false)
+			if err != nil {
+				return err
+			}
+			apiGroupVersion.TypeConverter = typeConverter
+		}
+
+		apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
+
+		discoveryAPIResources, r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
+
+		if err != nil {
+			return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
+		}
+		resourceInfos = append(resourceInfos, r...)
+
+		if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
+			// Aggregated discovery only aggregates resources under /apis
+			if apiPrefix == APIGroupPrefix {
+				s.AggregatedDiscoveryGroupManager.AddGroupVersion(
+					groupVersion.Group,
+					apidiscoveryv2beta1.APIVersionDiscovery{
+						Version:   groupVersion.Version,
+						Resources: discoveryAPIResources,
+					},
+				)
+			} else {
+				// There is only one group version for legacy resources, priority can be defaulted to 0.
+				s.AggregatedLegacyDiscoveryGroupManager.AddGroupVersion(
+					groupVersion.Group,
+					apidiscoveryv2beta1.APIVersionDiscovery{
+						Version:   groupVersion.Version,
+						Resources: discoveryAPIResources,
+					},
+				)
+			}
+		}
+
+	}
+
+	s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
+		utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
+		// API installation happens before we start listening on the handlers,
+		// therefore it is safe to register ResourceInfos here. The handler will block
+		// write requests until the storage versions of the targeting resources are updated.
+		s.StorageVersionManager.AddResourceInfo(resourceInfos...)
+	}
+
+	return nil
+}
+
+// InstallLegacyAPIGroup exposes the given legacy api group in the API.
+// The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the
+// underlying storage will be destroyed on this servers shutdown.
+func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo *APIGroupInfo) error {
+	if !s.legacyAPIGroupPrefixes.Has(apiPrefix) {
+		return fmt.Errorf("%q is not in the allowed legacy API prefixes: %v", apiPrefix, s.legacyAPIGroupPrefixes.List())
+	}
+
+	openAPIModels, err := s.getOpenAPIModels(apiPrefix, apiGroupInfo)
+	if err != nil {
+		return fmt.Errorf("unable to get openapi models: %v", err)
+	}
+
+	if err := s.installAPIResources(apiPrefix, apiGroupInfo, openAPIModels); err != nil {
+		return err
+	}
+
+	// Install the version handler.
+	// Add a handler at /<apiPrefix> to enumerate the supported api versions.
+	legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
+	if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
+		wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
+		s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
+	} else {
+		s.Handler.GoRestfulContainer.Add(legacyRootAPIHandler.WebService())
+	}
+
+	return nil
+}
+
+// InstallAPIGroups exposes given api groups in the API.
+// The <apiGroupInfos> passed into this function shouldn't be used elsewhere as the
+// underlying storage will be destroyed on this servers shutdown.
+func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) error {
+	for _, apiGroupInfo := range apiGroupInfos {
+		// Do not register empty group or empty version.  Doing so claims /apis/ for the wrong entity to be returned.
+		// Catching these here places the error  much closer to its origin
+		if len(apiGroupInfo.PrioritizedVersions[0].Group) == 0 {
+			return fmt.Errorf("cannot register handler with an empty group for %#v", *apiGroupInfo)
+		}
+		if len(apiGroupInfo.PrioritizedVersions[0].Version) == 0 {
+			return fmt.Errorf("cannot register handler with an empty version for %#v", *apiGroupInfo)
+		}
+	}
+
+	openAPIModels, err := s.getOpenAPIModels(APIGroupPrefix, apiGroupInfos...)
+	if err != nil {
+		return fmt.Errorf("unable to get openapi models: %v", err)
+	}
+
+	for _, apiGroupInfo := range apiGroupInfos {
+		if err := s.installAPIResources(APIGroupPrefix, apiGroupInfo, openAPIModels); err != nil {
+			return fmt.Errorf("unable to install api resources: %v", err)
+		}
+
+		// setup discovery
+		// Install the version handler.
+		// Add a handler at /apis/<groupName> to enumerate all versions supported by this group.
+		apiVersionsForDiscovery := []metav1.GroupVersionForDiscovery{}
+		for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
+			// Check the config to make sure that we elide versions that don't have any resources
+			if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
+				continue
+			}
+			apiVersionsForDiscovery = append(apiVersionsForDiscovery, metav1.GroupVersionForDiscovery{
+				GroupVersion: groupVersion.String(),
+				Version:      groupVersion.Version,
+			})
+		}
+		preferredVersionForDiscovery := metav1.GroupVersionForDiscovery{
+			GroupVersion: apiGroupInfo.PrioritizedVersions[0].String(),
+			Version:      apiGroupInfo.PrioritizedVersions[0].Version,
+		}
+		apiGroup := metav1.APIGroup{
+			Name:             apiGroupInfo.PrioritizedVersions[0].Group,
+			Versions:         apiVersionsForDiscovery,
+			PreferredVersion: preferredVersionForDiscovery,
+		}
+
+		s.DiscoveryGroupManager.AddGroup(apiGroup)
+		s.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.Serializer, apiGroup).WebService())
+	}
+	return nil
+}
+
+// InstallAPIGroup exposes the given api group in the API.
+// The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the
+// underlying storage will be destroyed on this servers shutdown.
+func (s *GenericAPIServer) InstallAPIGroup(apiGroupInfo *APIGroupInfo) error {
+	return s.InstallAPIGroups(apiGroupInfo)
+}
+
+func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion, apiPrefix string) (*genericapi.APIGroupVersion, error) {
+	storage := make(map[string]rest.Storage)
+	for k, v := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
+		if strings.ToLower(k) != k {
+			return nil, fmt.Errorf("resource names must be lowercase only, not %q", k)
+		}
+		storage[k] = v
+	}
+	version := s.newAPIGroupVersion(apiGroupInfo, groupVersion)
+	version.Root = apiPrefix
+	version.Storage = storage
+	return version, nil
+}
+
+func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion {
+	return &genericapi.APIGroupVersion{
+		GroupVersion:     groupVersion,
+		MetaGroupVersion: apiGroupInfo.MetaGroupVersion,
+
+		ParameterCodec:        apiGroupInfo.ParameterCodec,
+		Serializer:            apiGroupInfo.NegotiatedSerializer,
+		Creater:               apiGroupInfo.Scheme,
+		Convertor:             apiGroupInfo.Scheme,
+		ConvertabilityChecker: apiGroupInfo.Scheme,
+		UnsafeConvertor:       runtime.UnsafeObjectConvertor(apiGroupInfo.Scheme),
+		Defaulter:             apiGroupInfo.Scheme,
+		Typer:                 apiGroupInfo.Scheme,
+		Namer:                 runtime.Namer(meta.NewAccessor()),
+
+		EquivalentResourceRegistry: s.EquivalentResourceRegistry,
+
+		Admit:             s.admissionControl,
+		MinRequestTimeout: s.minRequestTimeout,
+		Authorizer:        s.Authorizer,
+	}
+}
+
+// NewDefaultAPIGroupInfo returns an APIGroupInfo stubbed with "normal" values
+// exposed for easier composition from other packages
+func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec runtime.ParameterCodec, codecs serializer.CodecFactory) APIGroupInfo {
+	return APIGroupInfo{
+		PrioritizedVersions:          scheme.PrioritizedVersionsForGroup(group),
+		VersionedResourcesStorageMap: map[string]map[string]rest.Storage{},
+		// TODO unhardcode this.  It was hardcoded before, but we need to re-evaluate
+		OptionsExternalVersion: &schema.GroupVersion{Version: "v1"},
+		Scheme:                 scheme,
+		ParameterCodec:         parameterCodec,
+		NegotiatedSerializer:   codecs,
+	}
+}
+
+// getOpenAPIModels is a private method for getting the OpenAPI models
+func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (openapiproto.Models, error) {
+	if s.openAPIConfig == nil {
+		return nil, nil
+	}
+	pathsToIgnore := openapiutil.NewTrie(s.openAPIConfig.IgnorePrefixes)
+	resourceNames := make([]string, 0)
+	for _, apiGroupInfo := range apiGroupInfos {
+		groupResources, err := getResourceNamesForGroup(apiPrefix, apiGroupInfo, pathsToIgnore)
+		if err != nil {
+			return nil, err
+		}
+		resourceNames = append(resourceNames, groupResources...)
+	}
+
+	// Build the openapi definitions for those resources and convert it to proto models
+	openAPISpec, err := openapibuilder2.BuildOpenAPIDefinitionsForResources(s.openAPIConfig, resourceNames...)
+	if err != nil {
+		return nil, err
+	}
+	for _, apiGroupInfo := range apiGroupInfos {
+		apiGroupInfo.StaticOpenAPISpec = openAPISpec
+	}
+	return utilopenapi.ToProtoModels(openAPISpec)
+}
+
+// getResourceNamesForGroup is a private method for getting the canonical names for each resource to build in an api group
+func getResourceNamesForGroup(apiPrefix string, apiGroupInfo *APIGroupInfo, pathsToIgnore openapiutil.Trie) ([]string, error) {
+	// Get the canonical names of every resource we need to build in this api group
+	resourceNames := make([]string, 0)
+	for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
+		for resource, storage := range apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version] {
+			path := gpath.Join(apiPrefix, groupVersion.Group, groupVersion.Version, resource)
+			if !pathsToIgnore.HasPrefix(path) {
+				kind, err := genericapi.GetResourceKind(groupVersion, storage, apiGroupInfo.Scheme)
+				if err != nil {
+					return nil, err
+				}
+				sampleObject, err := apiGroupInfo.Scheme.New(kind)
+				if err != nil {
+					return nil, err
+				}
+				name := openapiutil.GetCanonicalTypeName(sampleObject)
+				resourceNames = append(resourceNames, name)
+			}
+		}
+	}
+
+	return resourceNames, nil
+}

vendor/k8s.io/apiserver/pkg/server/handler.go

@@ -0,0 +1,190 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	rt "runtime"
+	"sort"
+	"strings"
+
+	"github.com/emicklei/go-restful/v3"
+	"k8s.io/klog/v2"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// APIServerHandlers holds the different http.Handlers used by the API server.
+// This includes the full handler chain, the director (which chooses between gorestful and nonGoRestful,
+// the gorestful handler (used for the API) which falls through to the nonGoRestful handler on unregistered paths,
+// and the nonGoRestful handler (which can contain a fallthrough of its own)
+// FullHandlerChain -> Director -> {GoRestfulContainer,NonGoRestfulMux} based on inspection of registered web services
+type APIServerHandler struct {
+	// FullHandlerChain is the one that is eventually served with.  It should include the full filter
+	// chain and then call the Director.
+	FullHandlerChain http.Handler
+	// The registered APIs.  InstallAPIs uses this.  Other servers probably shouldn't access this directly.
+	GoRestfulContainer *restful.Container
+	// NonGoRestfulMux is the final HTTP handler in the chain.
+	// It comes after all filters and the API handling
+	// This is where other servers can attach handler to various parts of the chain.
+	NonGoRestfulMux *mux.PathRecorderMux
+
+	// Director is here so that we can properly handle fall through and proxy cases.
+	// This looks a bit bonkers, but here's what's happening.  We need to have /apis handling registered in gorestful in order to have
+	// swagger generated for compatibility.  Doing that with `/apis` as a webservice, means that it forcibly 404s (no defaulting allowed)
+	// all requests which are not /apis or /apis/.  We need those calls to fall through behind goresful for proper delegation.  Trying to
+	// register for a pattern which includes everything behind it doesn't work because gorestful negotiates for verbs and content encoding
+	// and all those things go crazy when gorestful really just needs to pass through.  In addition, openapi enforces unique verb constraints
+	// which we don't fit into and it still muddies up swagger.  Trying to switch the webservices into a route doesn't work because the
+	//  containing webservice faces all the same problems listed above.
+	// This leads to the crazy thing done here.  Our mux does what we need, so we'll place it in front of gorestful.  It will introspect to
+	// decide if the route is likely to be handled by goresful and route there if needed.  Otherwise, it goes to NonGoRestfulMux mux in
+	// order to handle "normal" paths and delegation. Hopefully no API consumers will ever have to deal with this level of detail.  I think
+	// we should consider completely removing gorestful.
+	// Other servers should only use this opaquely to delegate to an API server.
+	Director http.Handler
+}
+
+// HandlerChainBuilderFn is used to wrap the GoRestfulContainer handler using the provided handler chain.
+// It is normally used to apply filtering like authentication and authorization
+type HandlerChainBuilderFn func(apiHandler http.Handler) http.Handler
+
+func NewAPIServerHandler(name string, s runtime.NegotiatedSerializer, handlerChainBuilder HandlerChainBuilderFn, notFoundHandler http.Handler) *APIServerHandler {
+	nonGoRestfulMux := mux.NewPathRecorderMux(name)
+	if notFoundHandler != nil {
+		nonGoRestfulMux.NotFoundHandler(notFoundHandler)
+	}
+
+	gorestfulContainer := restful.NewContainer()
+	gorestfulContainer.ServeMux = http.NewServeMux()
+	gorestfulContainer.Router(restful.CurlyRouter{}) // e.g. for proxy/{kind}/{name}/{*}
+	gorestfulContainer.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
+		logStackOnRecover(s, panicReason, httpWriter)
+	})
+	gorestfulContainer.ServiceErrorHandler(func(serviceErr restful.ServiceError, request *restful.Request, response *restful.Response) {
+		serviceErrorHandler(s, serviceErr, request, response)
+	})
+
+	director := director{
+		name:               name,
+		goRestfulContainer: gorestfulContainer,
+		nonGoRestfulMux:    nonGoRestfulMux,
+	}
+
+	return &APIServerHandler{
+		FullHandlerChain:   handlerChainBuilder(director),
+		GoRestfulContainer: gorestfulContainer,
+		NonGoRestfulMux:    nonGoRestfulMux,
+		Director:           director,
+	}
+}
+
+// ListedPaths returns the paths that should be shown under /
+func (a *APIServerHandler) ListedPaths() []string {
+	var handledPaths []string
+	// Extract the paths handled using restful.WebService
+	for _, ws := range a.GoRestfulContainer.RegisteredWebServices() {
+		handledPaths = append(handledPaths, ws.RootPath())
+	}
+	handledPaths = append(handledPaths, a.NonGoRestfulMux.ListedPaths()...)
+	sort.Strings(handledPaths)
+
+	return handledPaths
+}
+
+type director struct {
+	name               string
+	goRestfulContainer *restful.Container
+	nonGoRestfulMux    *mux.PathRecorderMux
+}
+
+func (d director) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	path := req.URL.Path
+
+	// check to see if our webservices want to claim this path
+	for _, ws := range d.goRestfulContainer.RegisteredWebServices() {
+		switch {
+		case ws.RootPath() == "/apis":
+			// if we are exactly /apis or /apis/, then we need special handling in loop.
+			// normally these are passed to the nonGoRestfulMux, but if discovery is enabled, it will go directly.
+			// We can't rely on a prefix match since /apis matches everything (see the big comment on Director above)
+			if path == "/apis" || path == "/apis/" {
+				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
+				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
+				// TODO fix gorestful, remove TPRs, or stop using gorestful
+				d.goRestfulContainer.Dispatch(w, req)
+				return
+			}
+
+		case strings.HasPrefix(path, ws.RootPath()):
+			// ensure an exact match or a path boundary match
+			if len(path) == len(ws.RootPath()) || path[len(ws.RootPath())] == '/' {
+				klog.V(5).Infof("%v: %v %q satisfied by gorestful with webservice %v", d.name, req.Method, path, ws.RootPath())
+				// don't use servemux here because gorestful servemuxes get messed up when removing webservices
+				// TODO fix gorestful, remove TPRs, or stop using gorestful
+				d.goRestfulContainer.Dispatch(w, req)
+				return
+			}
+		}
+	}
+
+	// if we didn't find a match, then we just skip gorestful altogether
+	klog.V(5).Infof("%v: %v %q satisfied by nonGoRestful", d.name, req.Method, path)
+	d.nonGoRestfulMux.ServeHTTP(w, req)
+}
+
+// TODO: Unify with RecoverPanics?
+func logStackOnRecover(s runtime.NegotiatedSerializer, panicReason interface{}, w http.ResponseWriter) {
+	var buffer bytes.Buffer
+	buffer.WriteString(fmt.Sprintf("recover from panic situation: - %v\r\n", panicReason))
+	for i := 2; ; i++ {
+		_, file, line, ok := rt.Caller(i)
+		if !ok {
+			break
+		}
+		buffer.WriteString(fmt.Sprintf("    %s:%d\r\n", file, line))
+	}
+	klog.Errorln(buffer.String())
+
+	headers := http.Header{}
+	if ct := w.Header().Get("Content-Type"); len(ct) > 0 {
+		headers.Set("Accept", ct)
+	}
+	responsewriters.ErrorNegotiated(apierrors.NewGenericServerResponse(http.StatusInternalServerError, "", schema.GroupResource{}, "", "", 0, false), s, schema.GroupVersion{}, w, &http.Request{Header: headers})
+}
+
+func serviceErrorHandler(s runtime.NegotiatedSerializer, serviceErr restful.ServiceError, request *restful.Request, resp *restful.Response) {
+	responsewriters.ErrorNegotiated(
+		apierrors.NewGenericServerResponse(serviceErr.Code, "", schema.GroupResource{}, "", serviceErr.Message, 0, false),
+		s,
+		schema.GroupVersion{},
+		resp,
+		request.Request,
+	)
+}
+
+// ServeHTTP makes it an http.Handler
+func (a *APIServerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	a.FullHandlerChain.ServeHTTP(w, r)
+}

vendor/k8s.io/apiserver/pkg/server/healthz.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/utils/clock"
+)
+
+// AddHealthChecks adds HealthCheck(s) to health endpoints (healthz, livez, readyz) but
+// configures the liveness grace period to be zero, which means we expect this health check
+// to immediately indicate that the apiserver is unhealthy.
+func (s *GenericAPIServer) AddHealthChecks(checks ...healthz.HealthChecker) error {
+	// we opt for a delay of zero here, because this entrypoint adds generic health checks
+	// and not health checks which are specifically related to kube-apiserver boot-sequences.
+	return s.addHealthChecks(0, checks...)
+}
+
+// AddBootSequenceHealthChecks adds health checks to the old healthz endpoint (for backwards compatibility reasons)
+// as well as livez and readyz. The livez grace period is defined by the value of the
+// command-line flag --livez-grace-period; before the grace period elapses, the livez health checks
+// will default to healthy. One may want to set a grace period in order to prevent the kubelet from restarting
+// the kube-apiserver due to long-ish boot sequences. Readyz health checks, on the other hand, have no grace period,
+// since readyz should fail until boot fully completes.
+func (s *GenericAPIServer) AddBootSequenceHealthChecks(checks ...healthz.HealthChecker) error {
+	return s.addHealthChecks(s.livezGracePeriod, checks...)
+}
+
+// addHealthChecks adds health checks to healthz, livez, and readyz. The delay passed in will set
+// a corresponding grace period on livez.
+func (s *GenericAPIServer) addHealthChecks(livezGracePeriod time.Duration, checks ...healthz.HealthChecker) error {
+	s.healthzLock.Lock()
+	defer s.healthzLock.Unlock()
+	if s.healthzChecksInstalled {
+		return fmt.Errorf("unable to add because the healthz endpoint has already been created")
+	}
+	s.healthzChecks = append(s.healthzChecks, checks...)
+	if err := s.AddLivezChecks(livezGracePeriod, checks...); err != nil {
+		return err
+	}
+	return s.AddReadyzChecks(checks...)
+}
+
+// AddReadyzChecks allows you to add a HealthCheck to readyz.
+func (s *GenericAPIServer) AddReadyzChecks(checks ...healthz.HealthChecker) error {
+	s.readyzLock.Lock()
+	defer s.readyzLock.Unlock()
+	if s.readyzChecksInstalled {
+		return fmt.Errorf("unable to add because the readyz endpoint has already been created")
+	}
+	s.readyzChecks = append(s.readyzChecks, checks...)
+	return nil
+}
+
+// AddLivezChecks allows you to add a HealthCheck to livez.
+func (s *GenericAPIServer) AddLivezChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	if s.livezChecksInstalled {
+		return fmt.Errorf("unable to add because the livez endpoint has already been created")
+	}
+	for _, check := range checks {
+		s.livezChecks = append(s.livezChecks, delayedHealthCheck(check, s.livezClock, delay))
+	}
+	return nil
+}
+
+// addReadyzShutdownCheck is a convenience function for adding a readyz shutdown check, so
+// that we can register that the api-server is no longer ready while we attempt to gracefully
+// shutdown.
+func (s *GenericAPIServer) addReadyzShutdownCheck(stopCh <-chan struct{}) error {
+	return s.AddReadyzChecks(shutdownCheck{stopCh})
+}
+
+// installHealthz creates the healthz endpoint for this server
+func (s *GenericAPIServer) installHealthz() {
+	s.healthzLock.Lock()
+	defer s.healthzLock.Unlock()
+	s.healthzChecksInstalled = true
+	healthz.InstallHandler(s.Handler.NonGoRestfulMux, s.healthzChecks...)
+}
+
+// installReadyz creates the readyz endpoint for this server.
+func (s *GenericAPIServer) installReadyz() {
+	s.readyzLock.Lock()
+	defer s.readyzLock.Unlock()
+	s.readyzChecksInstalled = true
+	healthz.InstallReadyzHandlerWithHealthyFunc(s.Handler.NonGoRestfulMux, func() {
+		// note: InstallReadyzHandlerWithHealthyFunc guarantees that this is called only once
+		s.lifecycleSignals.HasBeenReady.Signal()
+	}, s.readyzChecks...)
+}
+
+// installLivez creates the livez endpoint for this server.
+func (s *GenericAPIServer) installLivez() {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	s.livezChecksInstalled = true
+	healthz.InstallLivezHandler(s.Handler.NonGoRestfulMux, s.livezChecks...)
+}
+
+// shutdownCheck fails if the embedded channel is closed. This is intended to allow for graceful shutdown sequences
+// for the apiserver.
+type shutdownCheck struct {
+	StopCh <-chan struct{}
+}
+
+func (shutdownCheck) Name() string {
+	return "shutdown"
+}
+
+func (c shutdownCheck) Check(req *http.Request) error {
+	select {
+	case <-c.StopCh:
+		return fmt.Errorf("process is shutting down")
+	default:
+	}
+	return nil
+}
+
+// delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed. This
+// is intended for use primarily for livez health checks.
+func delayedHealthCheck(check healthz.HealthChecker, clock clock.Clock, delay time.Duration) healthz.HealthChecker {
+	return delayedLivezCheck{
+		check,
+		clock.Now().Add(delay),
+		clock,
+	}
+}
+
+type delayedLivezCheck struct {
+	check      healthz.HealthChecker
+	startCheck time.Time
+	clock      clock.Clock
+}
+
+func (c delayedLivezCheck) Name() string {
+	return c.check.Name()
+}
+
+func (c delayedLivezCheck) Check(req *http.Request) error {
+	if c.clock.Now().After(c.startCheck) {
+		return c.check.Check(req)
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/healthz/doc.go

@@ -0,0 +1,22 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package healthz implements basic http server health checking.
+// Usage:
+//
+//	import "k8s.io/apiserver/pkg/server/healthz"
+//	healthz.InstallHandler(mux)
+package healthz // import "k8s.io/apiserver/pkg/server/healthz"

vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -0,0 +1,314 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package healthz
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/component-base/metrics/prometheus/slis"
+	"k8s.io/klog/v2"
+)
+
+// HealthChecker is a named healthz checker.
+type HealthChecker interface {
+	Name() string
+	Check(req *http.Request) error
+}
+
+// PingHealthz returns true automatically when checked
+var PingHealthz HealthChecker = ping{}
+
+// ping implements the simplest possible healthz checker.
+type ping struct{}
+
+func (ping) Name() string {
+	return "ping"
+}
+
+// PingHealthz is a health check that returns true.
+func (ping) Check(_ *http.Request) error {
+	return nil
+}
+
+// LogHealthz returns true if logging is not blocked
+var LogHealthz HealthChecker = &log{}
+
+type log struct {
+	startOnce    sync.Once
+	lastVerified atomic.Value
+}
+
+func (l *log) Name() string {
+	return "log"
+}
+
+func (l *log) Check(_ *http.Request) error {
+	l.startOnce.Do(func() {
+		l.lastVerified.Store(time.Now())
+		go wait.Forever(func() {
+			klog.Flush()
+			l.lastVerified.Store(time.Now())
+		}, time.Minute)
+	})
+
+	lastVerified := l.lastVerified.Load().(time.Time)
+	if time.Since(lastVerified) < (2 * time.Minute) {
+		return nil
+	}
+	return fmt.Errorf("logging blocked")
+}
+
+type cacheSyncWaiter interface {
+	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
+}
+
+type informerSync struct {
+	cacheSyncWaiter cacheSyncWaiter
+}
+
+var _ HealthChecker = &informerSync{}
+
+// NewInformerSyncHealthz returns a new HealthChecker that will pass only if all informers in the given cacheSyncWaiter sync.
+func NewInformerSyncHealthz(cacheSyncWaiter cacheSyncWaiter) HealthChecker {
+	return &informerSync{
+		cacheSyncWaiter: cacheSyncWaiter,
+	}
+}
+
+func (i *informerSync) Name() string {
+	return "informer-sync"
+}
+
+func (i *informerSync) Check(_ *http.Request) error {
+	stopCh := make(chan struct{})
+	// Close stopCh to force checking if informers are synced now.
+	close(stopCh)
+
+	informersByStarted := make(map[bool][]string)
+	for informerType, started := range i.cacheSyncWaiter.WaitForCacheSync(stopCh) {
+		informersByStarted[started] = append(informersByStarted[started], informerType.String())
+	}
+
+	if notStarted := informersByStarted[false]; len(notStarted) > 0 {
+		return fmt.Errorf("%d informers not started yet: %v", len(notStarted), notStarted)
+	}
+	return nil
+}
+
+// NamedCheck returns a healthz checker for the given name and function.
+func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
+	return &healthzCheck{name, check}
+}
+
+// InstallHandler registers handlers for health checking on the path
+// "/healthz" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/healthz", checks...)
+}
+
+// InstallReadyzHandler registers handlers for health checking on the path
+// "/readyz" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/readyz", checks...)
+}
+
+// InstallReadyzHandlerWithHealthyFunc is like InstallReadyzHandler, but in addition call firstTimeReady
+// the first time /readyz succeeds.
+func InstallReadyzHandlerWithHealthyFunc(mux mux, firstTimeReady func(), checks ...HealthChecker) {
+	InstallPathHandlerWithHealthyFunc(mux, "/readyz", firstTimeReady, checks...)
+}
+
+// InstallLivezHandler registers handlers for liveness checking on the path
+// "/livez" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallLivezHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/livez", checks...)
+}
+
+// InstallPathHandler registers handlers for health checking on
+// a specific path to mux. *All handlers* for the path must be
+// specified in exactly one call to InstallPathHandler. Calling
+// InstallPathHandler more than once for the same path and mux will
+// result in a panic.
+func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
+	InstallPathHandlerWithHealthyFunc(mux, path, nil, checks...)
+}
+
+// InstallPathHandlerWithHealthyFunc is like InstallPathHandler, but calls firstTimeHealthy exactly once
+// when the handler succeeds for the first time.
+func InstallPathHandlerWithHealthyFunc(mux mux, path string, firstTimeHealthy func(), checks ...HealthChecker) {
+	if len(checks) == 0 {
+		klog.V(5).Info("No default health checks specified. Installing the ping handler.")
+		checks = []HealthChecker{PingHealthz}
+	}
+
+	klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
+
+	name := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
+	mux.Handle(path,
+		metrics.InstrumentHandlerFunc("GET",
+			/* group = */ "",
+			/* version = */ "",
+			/* resource = */ "",
+			/* subresource = */ path,
+			/* scope = */ "",
+			/* component = */ "",
+			/* deprecated */ false,
+			/* removedRelease */ "",
+			handleRootHealth(name, firstTimeHealthy, checks...)))
+	for _, check := range checks {
+		mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
+	}
+}
+
+// mux is an interface describing the methods InstallHandler requires.
+type mux interface {
+	Handle(pattern string, handler http.Handler)
+}
+
+// healthzCheck implements HealthChecker on an arbitrary name and check function.
+type healthzCheck struct {
+	name  string
+	check func(r *http.Request) error
+}
+
+var _ HealthChecker = &healthzCheck{}
+
+func (c *healthzCheck) Name() string {
+	return c.name
+}
+
+func (c *healthzCheck) Check(r *http.Request) error {
+	return c.check(r)
+}
+
+// getExcludedChecks extracts the health check names to be excluded from the query param
+func getExcludedChecks(r *http.Request) sets.String {
+	checks, found := r.URL.Query()["exclude"]
+	if found {
+		return sets.NewString(checks...)
+	}
+	return sets.NewString()
+}
+
+// handleRootHealth returns an http.HandlerFunc that serves the provided checks.
+func handleRootHealth(name string, firstTimeHealthy func(), checks ...HealthChecker) http.HandlerFunc {
+	var notifyOnce sync.Once
+	return func(w http.ResponseWriter, r *http.Request) {
+		excluded := getExcludedChecks(r)
+		// failedVerboseLogOutput is for output to the log.  It indicates detailed failed output information for the log.
+		var failedVerboseLogOutput bytes.Buffer
+		var failedChecks []string
+		var individualCheckOutput bytes.Buffer
+		for _, check := range checks {
+			// no-op the check if we've specified we want to exclude the check
+			if excluded.Has(check.Name()) {
+				excluded.Delete(check.Name())
+				fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
+				continue
+			}
+			if err := check.Check(r); err != nil {
+				slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Error)
+				// don't include the error since this endpoint is public.  If someone wants more detail
+				// they should have explicit permission to the detailed checks.
+				fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
+				// but we do want detailed information for our log
+				fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
+				failedChecks = append(failedChecks, check.Name())
+			} else {
+				slis.ObserveHealthcheck(context.Background(), check.Name(), name, slis.Success)
+				fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
+			}
+		}
+		if excluded.Len() > 0 {
+			fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
+			klog.V(6).Infof("cannot exclude some health checks, no health checks are installed matching %s",
+				formatQuoted(excluded.List()...))
+		}
+		// always be verbose on failure
+		if len(failedChecks) > 0 {
+			klog.V(2).Infof("%s check failed: %s\n%v", strings.Join(failedChecks, ","), name, failedVerboseLogOutput.String())
+			httplog.SetStacktracePredicate(r.Context(), func(int) bool { return false })
+			http.Error(w, fmt.Sprintf("%s%s check failed", individualCheckOutput.String(), name), http.StatusInternalServerError)
+			return
+		}
+
+		// signal first time this is healthy
+		if firstTimeHealthy != nil {
+			notifyOnce.Do(firstTimeHealthy)
+		}
+
+		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		if _, found := r.URL.Query()["verbose"]; !found {
+			fmt.Fprint(w, "ok")
+			return
+		}
+
+		individualCheckOutput.WriteTo(w)
+		fmt.Fprintf(w, "%s check passed\n", name)
+	}
+}
+
+// adaptCheckToHandler returns an http.HandlerFunc that serves the provided checks.
+func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := c(r)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
+		} else {
+			fmt.Fprint(w, "ok")
+		}
+	})
+}
+
+// checkerNames returns the names of the checks in the same order as passed in.
+func checkerNames(checks ...HealthChecker) []string {
+	// accumulate the names of checks for printing them out.
+	checkerNames := make([]string, 0, len(checks))
+	for _, check := range checks {
+		checkerNames = append(checkerNames, check.Name())
+	}
+	return checkerNames
+}
+
+// formatQuoted returns a formatted string of the health check names,
+// preserving the order passed in.
+func formatQuoted(names ...string) string {
+	quoted := make([]string, 0, len(names))
+	for _, name := range names {
+		quoted = append(quoted, fmt.Sprintf("%q", name))
+	}
+	return strings.Join(quoted, ",")
+}

vendor/k8s.io/apiserver/pkg/server/hooks.go

@@ -0,0 +1,245 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+	"runtime/debug"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/server/healthz"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
+)
+
+// PostStartHookFunc is a function that is called after the server has started.
+// It must properly handle cases like:
+//  1. asynchronous start in multiple API server processes
+//  2. conflicts between the different processes all trying to perform the same action
+//  3. partially complete work (API server crashes while running your hook)
+//  4. API server access **BEFORE** your hook has completed
+//
+// Think of it like a mini-controller that is super privileged and gets to run in-process
+// If you use this feature, tag @deads2k on github who has promised to review code for anyone's PostStartHook
+// until it becomes easier to use.
+type PostStartHookFunc func(context PostStartHookContext) error
+
+// PreShutdownHookFunc is a function that can be added to the shutdown logic.
+type PreShutdownHookFunc func() error
+
+// PostStartHookContext provides information about this API server to a PostStartHookFunc
+type PostStartHookContext struct {
+	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
+	LoopbackClientConfig *restclient.Config
+	// StopCh is the channel that will be closed when the server stops
+	StopCh <-chan struct{}
+}
+
+// PostStartHookProvider is an interface in addition to provide a post start hook for the api server
+type PostStartHookProvider interface {
+	PostStartHook() (string, PostStartHookFunc, error)
+}
+
+type postStartHookEntry struct {
+	hook PostStartHookFunc
+	// originatingStack holds the stack that registered postStartHooks. This allows us to show a more helpful message
+	// for duplicate registration.
+	originatingStack string
+
+	// done will be closed when the postHook is finished
+	done chan struct{}
+}
+
+type PostStartHookConfigEntry struct {
+	hook PostStartHookFunc
+	// originatingStack holds the stack that registered postStartHooks. This allows us to show a more helpful message
+	// for duplicate registration.
+	originatingStack string
+}
+
+type preShutdownHookEntry struct {
+	hook PreShutdownHookFunc
+}
+
+// AddPostStartHook allows you to add a PostStartHook.
+func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc) error {
+	if len(name) == 0 {
+		return fmt.Errorf("missing name")
+	}
+	if hook == nil {
+		return fmt.Errorf("hook func may not be nil: %q", name)
+	}
+	if s.disabledPostStartHooks.Has(name) {
+		klog.V(1).Infof("skipping %q because it was explicitly disabled", name)
+		return nil
+	}
+
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+
+	if s.postStartHooksCalled {
+		return fmt.Errorf("unable to add %q because PostStartHooks have already been called", name)
+	}
+	if postStartHook, exists := s.postStartHooks[name]; exists {
+		// this is programmer error, but it can be hard to debug
+		return fmt.Errorf("unable to add %q because it was already registered by: %s", name, postStartHook.originatingStack)
+	}
+
+	// done is closed when the poststarthook is finished.  This is used by the health check to be able to indicate
+	// that the poststarthook is finished
+	done := make(chan struct{})
+	if err := s.AddBootSequenceHealthChecks(postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
+		return err
+	}
+	s.postStartHooks[name] = postStartHookEntry{hook: hook, originatingStack: string(debug.Stack()), done: done}
+
+	return nil
+}
+
+// AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure
+func (s *GenericAPIServer) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
+	if err := s.AddPostStartHook(name, hook); err != nil {
+		klog.Fatalf("Error registering PostStartHook %q: %v", name, err)
+	}
+}
+
+// AddPreShutdownHook allows you to add a PreShutdownHook.
+func (s *GenericAPIServer) AddPreShutdownHook(name string, hook PreShutdownHookFunc) error {
+	if len(name) == 0 {
+		return fmt.Errorf("missing name")
+	}
+	if hook == nil {
+		return nil
+	}
+
+	s.preShutdownHookLock.Lock()
+	defer s.preShutdownHookLock.Unlock()
+
+	if s.preShutdownHooksCalled {
+		return fmt.Errorf("unable to add %q because PreShutdownHooks have already been called", name)
+	}
+	if _, exists := s.preShutdownHooks[name]; exists {
+		return fmt.Errorf("unable to add %q because it is already registered", name)
+	}
+
+	s.preShutdownHooks[name] = preShutdownHookEntry{hook: hook}
+
+	return nil
+}
+
+// AddPreShutdownHookOrDie allows you to add a PostStartHook, but dies on failure
+func (s *GenericAPIServer) AddPreShutdownHookOrDie(name string, hook PreShutdownHookFunc) {
+	if err := s.AddPreShutdownHook(name, hook); err != nil {
+		klog.Fatalf("Error registering PreShutdownHook %q: %v", name, err)
+	}
+}
+
+// RunPostStartHooks runs the PostStartHooks for the server
+func (s *GenericAPIServer) RunPostStartHooks(stopCh <-chan struct{}) {
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+	s.postStartHooksCalled = true
+
+	context := PostStartHookContext{
+		LoopbackClientConfig: s.LoopbackClientConfig,
+		StopCh:               stopCh,
+	}
+
+	for hookName, hookEntry := range s.postStartHooks {
+		go runPostStartHook(hookName, hookEntry, context)
+	}
+}
+
+// RunPreShutdownHooks runs the PreShutdownHooks for the server
+func (s *GenericAPIServer) RunPreShutdownHooks() error {
+	var errorList []error
+
+	s.preShutdownHookLock.Lock()
+	defer s.preShutdownHookLock.Unlock()
+	s.preShutdownHooksCalled = true
+
+	for hookName, hookEntry := range s.preShutdownHooks {
+		if err := runPreShutdownHook(hookName, hookEntry); err != nil {
+			errorList = append(errorList, err)
+		}
+	}
+	return utilerrors.NewAggregate(errorList)
+}
+
+// isPostStartHookRegistered checks whether a given PostStartHook is registered
+func (s *GenericAPIServer) isPostStartHookRegistered(name string) bool {
+	s.postStartHookLock.Lock()
+	defer s.postStartHookLock.Unlock()
+	_, exists := s.postStartHooks[name]
+	return exists
+}
+
+func runPostStartHook(name string, entry postStartHookEntry, context PostStartHookContext) {
+	var err error
+	func() {
+		// don't let the hook *accidentally* panic and kill the server
+		defer utilruntime.HandleCrash()
+		err = entry.hook(context)
+	}()
+	// if the hook intentionally wants to kill server, let it.
+	if err != nil {
+		klog.Fatalf("PostStartHook %q failed: %v", name, err)
+	}
+	close(entry.done)
+}
+
+func runPreShutdownHook(name string, entry preShutdownHookEntry) error {
+	var err error
+	func() {
+		// don't let the hook *accidentally* panic and kill the server
+		defer utilruntime.HandleCrash()
+		err = entry.hook()
+	}()
+	if err != nil {
+		return fmt.Errorf("PreShutdownHook %q failed: %v", name, err)
+	}
+	return nil
+}
+
+// postStartHookHealthz implements a healthz check for poststarthooks.  It will return a "hookNotFinished"
+// error until the poststarthook is finished.
+type postStartHookHealthz struct {
+	name string
+
+	// done will be closed when the postStartHook is finished
+	done chan struct{}
+}
+
+var _ healthz.HealthChecker = postStartHookHealthz{}
+
+func (h postStartHookHealthz) Name() string {
+	return h.name
+}
+
+var errHookNotFinished = errors.New("not finished")
+
+func (h postStartHookHealthz) Check(req *http.Request) error {
+	select {
+	case <-h.done:
+		return nil
+	default:
+		return errHookNotFinished
+	}
+}

vendor/k8s.io/apiserver/pkg/server/httplog/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package httplog contains a helper object and functions to maintain a log
+// along with an http response.
+package httplog // import "k8s.io/apiserver/pkg/server/httplog"

vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -0,0 +1,326 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package httplog
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/klog/v2"
+)
+
+// StacktracePred returns true if a stacktrace should be logged for this status.
+type StacktracePred func(httpStatus int) (logStacktrace bool)
+
+// ShouldLogRequestPred returns true if logging should be enabled for this request
+type ShouldLogRequestPred func() bool
+
+type logger interface {
+	Addf(format string, data ...interface{})
+}
+
+type respLoggerContextKeyType int
+
+// respLoggerContextKey is used to store the respLogger pointer in the request context.
+const respLoggerContextKey respLoggerContextKeyType = iota
+
+// Add a layer on top of ResponseWriter, so we can track latency and error
+// message sources.
+//
+// TODO now that we're using go-restful, we shouldn't need to be wrapping
+// the http.ResponseWriter. We can recover panics from go-restful, and
+// the logging value is questionable.
+type respLogger struct {
+	hijacked       bool
+	statusRecorded bool
+	status         int
+	statusStack    string
+	// mutex is used when accessing addedInfo, addedKeyValuePairs and logStacktracePred.
+	// They can be modified by other goroutine when logging happens (in case of request timeout)
+	mutex              sync.Mutex
+	addedInfo          strings.Builder
+	addedKeyValuePairs []interface{}
+	startTime          time.Time
+
+	captureErrorOutput bool
+
+	req       *http.Request
+	userAgent string
+	w         http.ResponseWriter
+
+	logStacktracePred StacktracePred
+}
+
+var _ http.ResponseWriter = &respLogger{}
+var _ responsewriter.UserProvidedDecorator = &respLogger{}
+
+func (rl *respLogger) Unwrap() http.ResponseWriter {
+	return rl.w
+}
+
+// Simple logger that logs immediately when Addf is called
+type passthroughLogger struct{}
+
+// Addf logs info immediately.
+func (passthroughLogger) Addf(format string, data ...interface{}) {
+	klog.V(2).Info(fmt.Sprintf(format, data...))
+}
+
+// DefaultStacktracePred is the default implementation of StacktracePred.
+func DefaultStacktracePred(status int) bool {
+	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
+}
+
+const withLoggingLevel = 3
+
+// WithLogging wraps the handler with logging.
+func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
+	return withLogging(handler, pred, func() bool {
+		return klog.V(withLoggingLevel).Enabled()
+	})
+}
+
+func withLogging(handler http.Handler, stackTracePred StacktracePred, shouldLogRequest ShouldLogRequestPred) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if !shouldLogRequest() {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		ctx := req.Context()
+		if old := respLoggerFromRequest(req); old != nil {
+			panic("multiple WithLogging calls!")
+		}
+
+		startTime := time.Now()
+		if receivedTimestamp, ok := request.ReceivedTimestampFrom(ctx); ok {
+			startTime = receivedTimestamp
+		}
+
+		rl := newLoggedWithStartTime(req, w, startTime)
+		rl.StacktraceWhen(stackTracePred)
+		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
+		defer rl.Log()
+
+		w = responsewriter.WrapForHTTP1Or2(rl)
+		handler.ServeHTTP(w, req)
+	})
+}
+
+// respLoggerFromContext returns the respLogger or nil.
+func respLoggerFromContext(ctx context.Context) *respLogger {
+	val := ctx.Value(respLoggerContextKey)
+	if rl, ok := val.(*respLogger); ok {
+		return rl
+	}
+	return nil
+}
+
+func respLoggerFromRequest(req *http.Request) *respLogger {
+	return respLoggerFromContext(req.Context())
+}
+
+func newLoggedWithStartTime(req *http.Request, w http.ResponseWriter, startTime time.Time) *respLogger {
+	logger := &respLogger{
+		startTime:         startTime,
+		req:               req,
+		userAgent:         req.UserAgent(),
+		w:                 w,
+		logStacktracePred: DefaultStacktracePred,
+	}
+	return logger
+}
+
+// newLogged turns a normal response writer into a logged response writer.
+func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
+	return newLoggedWithStartTime(req, w, time.Now())
+}
+
+// LogOf returns the logger hiding in w. If there is not an existing logger
+// then a passthroughLogger will be created which will log to stdout immediately
+// when Addf is called.
+func LogOf(req *http.Request, w http.ResponseWriter) logger {
+	if rl := respLoggerFromRequest(req); rl != nil {
+		return rl
+	}
+	return &passthroughLogger{}
+}
+
+// Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
+func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
+	if rl := respLoggerFromRequest(req); rl != nil {
+		return rl.w
+	}
+	return w
+}
+
+// StacktraceWhen sets the stacktrace logging predicate, which decides when to log a stacktrace.
+// There's a default, so you don't need to call this unless you don't like the default.
+func (rl *respLogger) StacktraceWhen(pred StacktracePred) *respLogger {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.logStacktracePred = pred
+	return rl
+}
+
+// StatusIsNot returns a StacktracePred which will cause stacktraces to be logged
+// for any status *not* in the given list.
+func StatusIsNot(statuses ...int) StacktracePred {
+	statusesNoTrace := map[int]bool{}
+	for _, s := range statuses {
+		statusesNoTrace[s] = true
+	}
+	return func(status int) bool {
+		_, ok := statusesNoTrace[status]
+		return !ok
+	}
+}
+
+// Addf adds additional data to be logged with this request.
+func (rl *respLogger) Addf(format string, data ...interface{}) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.addedInfo.WriteString("\n")
+	rl.addedInfo.WriteString(fmt.Sprintf(format, data...))
+}
+
+func AddInfof(ctx context.Context, format string, data ...interface{}) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.Addf(format, data...)
+	}
+}
+
+func (rl *respLogger) AddKeyValue(key string, value interface{}) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.addedKeyValuePairs = append(rl.addedKeyValuePairs, key, value)
+}
+
+// AddKeyValue adds a (key, value) pair to the httplog associated
+// with the request.
+// Use this function if you want your data to show up in httplog
+// in a more structured and readable way.
+func AddKeyValue(ctx context.Context, key string, value interface{}) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.AddKeyValue(key, value)
+	}
+}
+
+// SetStacktracePredicate sets a custom stacktrace predicate for the
+// logger associated with the given request context.
+func SetStacktracePredicate(ctx context.Context, pred StacktracePred) {
+	if rl := respLoggerFromContext(ctx); rl != nil {
+		rl.StacktraceWhen(pred)
+	}
+}
+
+// Log is intended to be called once at the end of your request handler, via defer
+func (rl *respLogger) Log() {
+	latency := time.Since(rl.startTime)
+	auditID := audit.GetAuditIDTruncated(rl.req.Context())
+	verb := metrics.NormalizedVerb(rl.req)
+
+	keysAndValues := []interface{}{
+		"verb", verb,
+		"URI", rl.req.RequestURI,
+		"latency", latency,
+		// We can't get UserAgent from rl.req.UserAgent() here as it accesses headers map,
+		// which can be modified in another goroutine when apiserver request times out.
+		// For example authentication filter modifies request's headers,
+		// This can cause apiserver to crash with unrecoverable fatal error.
+		// More info about concurrent read and write for maps: https://golang.org/doc/go1.6#runtime
+		"userAgent", rl.userAgent,
+		"audit-ID", auditID,
+		"srcIP", rl.req.RemoteAddr,
+	}
+	// Lock for accessing addedKeyValuePairs and addedInfo
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	keysAndValues = append(keysAndValues, rl.addedKeyValuePairs...)
+
+	if rl.hijacked {
+		keysAndValues = append(keysAndValues, "hijacked", true)
+	} else {
+		keysAndValues = append(keysAndValues, "resp", rl.status)
+		if len(rl.statusStack) > 0 {
+			keysAndValues = append(keysAndValues, "statusStack", rl.statusStack)
+		}
+		info := rl.addedInfo.String()
+		if len(info) > 0 {
+			keysAndValues = append(keysAndValues, "addedInfo", info)
+		}
+	}
+
+	klog.V(withLoggingLevel).InfoSDepth(1, "HTTP", keysAndValues...)
+}
+
+// Header implements http.ResponseWriter.
+func (rl *respLogger) Header() http.Header {
+	return rl.w.Header()
+}
+
+// Write implements http.ResponseWriter.
+func (rl *respLogger) Write(b []byte) (int, error) {
+	if !rl.statusRecorded {
+		rl.recordStatus(http.StatusOK) // Default if WriteHeader hasn't been called
+	}
+	if rl.captureErrorOutput {
+		rl.Addf("logging error output: %q\n", string(b))
+	}
+	return rl.w.Write(b)
+}
+
+// WriteHeader implements http.ResponseWriter.
+func (rl *respLogger) WriteHeader(status int) {
+	rl.recordStatus(status)
+	rl.w.WriteHeader(status)
+}
+
+func (rl *respLogger) Hijack() (net.Conn, *bufio.ReadWriter, error) {
+	rl.hijacked = true
+
+	// the outer ResponseWriter object returned by WrapForHTTP1Or2 implements
+	// http.Hijacker if the inner object (rl.w) implements http.Hijacker.
+	return rl.w.(http.Hijacker).Hijack()
+}
+
+func (rl *respLogger) recordStatus(status int) {
+	rl.mutex.Lock()
+	defer rl.mutex.Unlock()
+	rl.status = status
+	rl.statusRecorded = true
+	if rl.logStacktracePred(status) {
+		// Only log stacks for errors
+		stack := make([]byte, 50*1024)
+		stack = stack[:runtime.Stack(stack, false)]
+		rl.statusStack = "\n" + string(stack)
+		rl.captureErrorOutput = true
+	} else {
+		rl.statusStack = ""
+	}
+}

vendor/k8s.io/apiserver/pkg/server/lifecycle_signals.go

@@ -0,0 +1,190 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"sync"
+)
+
+/*
+We make an attempt here to identify the events that take place during
+lifecycle of the apiserver.
+
+We also identify each event with a name so we can refer to it.
+
+Events:
+- ShutdownInitiated: KILL signal received
+- AfterShutdownDelayDuration: shutdown delay duration has passed
+- InFlightRequestsDrained: all in flight request(s) have been drained
+- HasBeenReady is signaled when the readyz endpoint succeeds for the first time
+
+The following is a sequence of shutdown events that we expect to see with
+  'ShutdownSendRetryAfter' = false:
+
+T0: ShutdownInitiated: KILL signal received
+	- /readyz starts returning red
+    - run pre shutdown hooks
+
+T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
+	- the default value of 'ShutdownDelayDuration' is '70s'
+	- it's time to initiate shutdown of the HTTP Server, server.Shutdown is invoked
+	- as a consequene, the Close function has is called for all listeners
+ 	- the HTTP Server stops listening immediately
+	- any new request arriving on a new TCP socket is denied with
+      a network error similar to 'connection refused'
+    - the HTTP Server waits gracefully for existing requests to complete
+      up to '60s' (dictated by ShutdownTimeout)
+	- active long running requests will receive a GOAWAY.
+
+T0+70s: HTTPServerStoppedListening:
+	- this event is signaled when the HTTP Server has stopped listening
+      which is immediately after server.Shutdown has been invoked
+
+T0 + 70s + up-to 60s: InFlightRequestsDrained: existing in flight requests have been drained
+	- long running requests are outside of this scope
+	- up-to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
+      any request in flight has a hard timeout of 60s.
+	- it's time to call 'Shutdown' on the audit events since all
+	  in flight request(s) have drained.
+
+
+The following is a sequence of shutdown events that we expect to see with
+  'ShutdownSendRetryAfter' = true:
+
+T0: ShutdownInitiated: KILL signal received
+	- /readyz starts returning red
+    - run pre shutdown hooks
+
+T0+70s: AfterShutdownDelayDuration: shutdown delay duration has passed
+	- the default value of 'ShutdownDelayDuration' is '70s'
+	- the HTTP Server will continue to listen
+	- the apiserver is not accepting new request(s)
+		- it includes new request(s) on a new or an existing TCP connection
+		- new request(s) arriving after this point are replied with a 429
+      	  and the  response headers: 'Retry-After: 1` and 'Connection: close'
+	- note: these new request(s) will not show up in audit logs
+
+T0 + 70s + up to 60s: InFlightRequestsDrained: existing in flight requests have been drained
+	- long running requests are outside of this scope
+	- up to 60s: the default value of 'ShutdownTimeout' is 60s, this means that
+      any request in flight has a hard timeout of 60s.
+	- server.Shutdown is called, the HTTP Server stops listening immediately
+    - the HTTP Server waits gracefully for existing requests to complete
+      up to '2s' (it's hard coded right now)
+*/
+
+// lifecycleSignal encapsulates a named apiserver event
+type lifecycleSignal interface {
+	// Signal signals the event, indicating that the event has occurred.
+	// Signal is idempotent, once signaled the event stays signaled and
+	// it immediately unblocks any goroutine waiting for this event.
+	Signal()
+
+	// Signaled returns a channel that is closed when the underlying event
+	// has been signaled. Successive calls to Signaled return the same value.
+	Signaled() <-chan struct{}
+
+	// Name returns the name of the signal, useful for logging.
+	Name() string
+}
+
+// lifecycleSignals provides an abstraction of the events that
+// transpire during the lifecycle of the apiserver. This abstraction makes it easy
+// for us to write unit tests that can verify expected graceful termination behavior.
+//
+// GenericAPIServer can use these to either:
+//   - signal that a particular termination event has transpired
+//   - wait for a designated termination event to transpire and do some action.
+type lifecycleSignals struct {
+	// ShutdownInitiated event is signaled when an apiserver shutdown has been initiated.
+	// It is signaled when the `stopCh` provided by the main goroutine
+	// receives a KILL signal and is closed as a consequence.
+	ShutdownInitiated lifecycleSignal
+
+	// AfterShutdownDelayDuration event is signaled as soon as ShutdownDelayDuration
+	// has elapsed since the ShutdownInitiated event.
+	// ShutdownDelayDuration allows the apiserver to delay shutdown for some time.
+	AfterShutdownDelayDuration lifecycleSignal
+
+	// PreShutdownHooksStopped event is signaled when all registered
+	// preshutdown hook(s) have finished running.
+	PreShutdownHooksStopped lifecycleSignal
+
+	// NotAcceptingNewRequest event is signaled when the server is no
+	// longer accepting any new request, from this point on any new
+	// request will receive an error.
+	NotAcceptingNewRequest lifecycleSignal
+
+	// InFlightRequestsDrained event is signaled when the existing requests
+	// in flight have completed. This is used as signal to shut down the audit backends
+	InFlightRequestsDrained lifecycleSignal
+
+	// HTTPServerStoppedListening termination event is signaled when the
+	// HTTP Server has stopped listening to the underlying socket.
+	HTTPServerStoppedListening lifecycleSignal
+
+	// HasBeenReady is signaled when the readyz endpoint succeeds for the first time.
+	HasBeenReady lifecycleSignal
+
+	// MuxAndDiscoveryComplete is signaled when all known HTTP paths have been installed.
+	// It exists primarily to avoid returning a 404 response when a resource actually exists but we haven't installed the path to a handler.
+	// The actual logic is implemented by an APIServer using the generic server library.
+	MuxAndDiscoveryComplete lifecycleSignal
+}
+
+// newLifecycleSignals returns an instance of lifecycleSignals interface to be used
+// to coordinate lifecycle of the apiserver
+func newLifecycleSignals() lifecycleSignals {
+	return lifecycleSignals{
+		ShutdownInitiated:          newNamedChannelWrapper("ShutdownInitiated"),
+		AfterShutdownDelayDuration: newNamedChannelWrapper("AfterShutdownDelayDuration"),
+		PreShutdownHooksStopped:    newNamedChannelWrapper("PreShutdownHooksStopped"),
+		NotAcceptingNewRequest:     newNamedChannelWrapper("NotAcceptingNewRequest"),
+		InFlightRequestsDrained:    newNamedChannelWrapper("InFlightRequestsDrained"),
+		HTTPServerStoppedListening: newNamedChannelWrapper("HTTPServerStoppedListening"),
+		HasBeenReady:               newNamedChannelWrapper("HasBeenReady"),
+		MuxAndDiscoveryComplete:    newNamedChannelWrapper("MuxAndDiscoveryComplete"),
+	}
+}
+
+func newNamedChannelWrapper(name string) lifecycleSignal {
+	return &namedChannelWrapper{
+		name: name,
+		once: sync.Once{},
+		ch:   make(chan struct{}),
+	}
+}
+
+type namedChannelWrapper struct {
+	name string
+	once sync.Once
+	ch   chan struct{}
+}
+
+func (e *namedChannelWrapper) Signal() {
+	e.once.Do(func() {
+		close(e.ch)
+	})
+}
+
+func (e *namedChannelWrapper) Signaled() <-chan struct{} {
+	return e.ch
+}
+
+func (e *namedChannelWrapper) Name() string {
+	return e.name
+}

vendor/k8s.io/apiserver/pkg/server/mux/OWNERS

@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts

vendor/k8s.io/apiserver/pkg/server/mux/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package mux contains abstractions for http multiplexing of APIs.
+package mux // import "k8s.io/apiserver/pkg/server/mux"

vendor/k8s.io/apiserver/pkg/server/mux/pathrecorder.go

@@ -0,0 +1,279 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mux
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"sort"
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// PathRecorderMux wraps a mux object and records the registered exposedPaths.
+type PathRecorderMux struct {
+	// name is used for logging so you can trace requests through
+	name string
+
+	lock            sync.Mutex
+	notFoundHandler http.Handler
+	pathToHandler   map[string]http.Handler
+	prefixToHandler map[string]http.Handler
+
+	// mux stores a pathHandler and is used to handle the actual serving.
+	// Turns out, we want to accept trailing slashes, BUT we don't care about handling
+	// everything under them.  This does exactly matches only unless its explicitly requested to
+	// do something different
+	mux atomic.Value
+
+	// exposedPaths is the list of paths that should be shown at /
+	exposedPaths []string
+
+	// pathStacks holds the stacks of all registered paths.  This allows us to show a more helpful message
+	// before the "http: multiple registrations for %s" panic.
+	pathStacks map[string]string
+}
+
+// pathHandler is an http.Handler that will satisfy requests first by exact match, then by prefix,
+// then by notFoundHandler
+type pathHandler struct {
+	// muxName is used for logging so you can trace requests through
+	muxName string
+
+	// pathToHandler is a map of exactly matching request to its handler
+	pathToHandler map[string]http.Handler
+
+	// this has to be sorted by most slashes then by length
+	prefixHandlers []prefixHandler
+
+	// notFoundHandler is the handler to use for satisfying requests with no other match
+	notFoundHandler http.Handler
+}
+
+// prefixHandler holds the prefix it should match and the handler to use
+type prefixHandler struct {
+	// prefix is the prefix to test for a request match
+	prefix string
+	// handler is used to satisfy matching requests
+	handler http.Handler
+}
+
+// NewPathRecorderMux creates a new PathRecorderMux
+func NewPathRecorderMux(name string) *PathRecorderMux {
+	ret := &PathRecorderMux{
+		name:            name,
+		pathToHandler:   map[string]http.Handler{},
+		prefixToHandler: map[string]http.Handler{},
+		mux:             atomic.Value{},
+		exposedPaths:    []string{},
+		pathStacks:      map[string]string{},
+	}
+
+	ret.mux.Store(&pathHandler{notFoundHandler: http.NotFoundHandler()})
+	return ret
+}
+
+// ListedPaths returns the registered handler exposedPaths.
+func (m *PathRecorderMux) ListedPaths() []string {
+	handledPaths := append([]string{}, m.exposedPaths...)
+	sort.Strings(handledPaths)
+
+	return handledPaths
+}
+
+func (m *PathRecorderMux) trackCallers(path string) {
+	stack := string(debug.Stack())
+	if existingStack, ok := m.pathStacks[path]; ok {
+		utilruntime.HandleError(fmt.Errorf("duplicate path registration of %q: original registration from %v\n\nnew registration from %v", path, existingStack, stack))
+	}
+	m.pathStacks[path] = stack
+}
+
+// refreshMuxLocked creates a new mux and must be called while locked.  Otherwise the view of handlers may
+// not be consistent
+func (m *PathRecorderMux) refreshMuxLocked() {
+	newMux := &pathHandler{
+		muxName:         m.name,
+		pathToHandler:   map[string]http.Handler{},
+		prefixHandlers:  []prefixHandler{},
+		notFoundHandler: http.NotFoundHandler(),
+	}
+	if m.notFoundHandler != nil {
+		newMux.notFoundHandler = m.notFoundHandler
+	}
+	for path, handler := range m.pathToHandler {
+		newMux.pathToHandler[path] = handler
+	}
+
+	keys := sets.StringKeySet(m.prefixToHandler).List()
+	sort.Sort(sort.Reverse(byPrefixPriority(keys)))
+	for _, prefix := range keys {
+		newMux.prefixHandlers = append(newMux.prefixHandlers, prefixHandler{
+			prefix:  prefix,
+			handler: m.prefixToHandler[prefix],
+		})
+	}
+
+	m.mux.Store(newMux)
+}
+
+// NotFoundHandler sets the handler to use if there's no match for a give path
+func (m *PathRecorderMux) NotFoundHandler(notFoundHandler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	m.notFoundHandler = notFoundHandler
+
+	m.refreshMuxLocked()
+}
+
+// Unregister removes a path from the mux.
+func (m *PathRecorderMux) Unregister(path string) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+
+	delete(m.pathToHandler, path)
+	delete(m.prefixToHandler, path)
+	delete(m.pathStacks, path)
+	for i := range m.exposedPaths {
+		if m.exposedPaths[i] == path {
+			m.exposedPaths = append(m.exposedPaths[:i], m.exposedPaths[i+1:]...)
+			break
+		}
+	}
+
+	m.refreshMuxLocked()
+}
+
+// Handle registers the handler for the given pattern.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) Handle(path string, handler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.exposedPaths = append(m.exposedPaths, path)
+	m.pathToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// HandleFunc registers the handler function for the given pattern.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) HandleFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	m.Handle(path, http.HandlerFunc(handler))
+}
+
+// UnlistedHandle registers the handler for the given pattern, but doesn't list it.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) UnlistedHandle(path string, handler http.Handler) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.pathToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// UnlistedHandleFunc registers the handler function for the given pattern, but doesn't list it.
+// If a handler already exists for pattern, Handle panics.
+func (m *PathRecorderMux) UnlistedHandleFunc(path string, handler func(http.ResponseWriter, *http.Request)) {
+	m.UnlistedHandle(path, http.HandlerFunc(handler))
+}
+
+// HandlePrefix is like Handle, but matches for anything under the path.  Like a standard golang trailing slash.
+func (m *PathRecorderMux) HandlePrefix(path string, handler http.Handler) {
+	if !strings.HasSuffix(path, "/") {
+		panic(fmt.Sprintf("%q must end in a trailing slash", path))
+	}
+
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.exposedPaths = append(m.exposedPaths, path)
+	m.prefixToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// UnlistedHandlePrefix is like UnlistedHandle, but matches for anything under the path.  Like a standard golang trailing slash.
+func (m *PathRecorderMux) UnlistedHandlePrefix(path string, handler http.Handler) {
+	if !strings.HasSuffix(path, "/") {
+		panic(fmt.Sprintf("%q must end in a trailing slash", path))
+	}
+
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.trackCallers(path)
+
+	m.prefixToHandler[path] = handler
+	m.refreshMuxLocked()
+}
+
+// ServeHTTP makes it an http.Handler
+func (m *PathRecorderMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	m.mux.Load().(*pathHandler).ServeHTTP(w, r)
+}
+
+// ServeHTTP makes it an http.Handler
+func (h *pathHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if exactHandler, ok := h.pathToHandler[r.URL.Path]; ok {
+		klog.V(5).Infof("%v: %q satisfied by exact match", h.muxName, r.URL.Path)
+		exactHandler.ServeHTTP(w, r)
+		return
+	}
+
+	for _, prefixHandler := range h.prefixHandlers {
+		if strings.HasPrefix(r.URL.Path, prefixHandler.prefix) {
+			klog.V(5).Infof("%v: %q satisfied by prefix %v", h.muxName, r.URL.Path, prefixHandler.prefix)
+			prefixHandler.handler.ServeHTTP(w, r)
+			return
+		}
+	}
+
+	klog.V(5).Infof("%v: %q satisfied by NotFoundHandler", h.muxName, r.URL.Path)
+	h.notFoundHandler.ServeHTTP(w, r)
+}
+
+// byPrefixPriority sorts url prefixes by the order in which they should be tested by the mux
+// this has to be sorted by most slashes then by length so that we can iterate straight
+// through to match the "best" one first.
+type byPrefixPriority []string
+
+func (s byPrefixPriority) Len() int      { return len(s) }
+func (s byPrefixPriority) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byPrefixPriority) Less(i, j int) bool {
+	lhsNumParts := strings.Count(s[i], "/")
+	rhsNumParts := strings.Count(s[j], "/")
+	if lhsNumParts != rhsNumParts {
+		return lhsNumParts < rhsNumParts
+	}
+
+	lhsLen := len(s[i])
+	rhsLen := len(s[j])
+	if lhsLen != rhsLen {
+		return lhsLen < rhsLen
+	}
+
+	return strings.Compare(s[i], s[j]) < 0
+}

vendor/k8s.io/apiserver/pkg/server/options/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - smarterclayton
+  - wojtek-t
+  - deads2k
+  - liggitt
+  - sttts
+  - soltysh
+  - dims
+  - cjcullen
+  - ping035627
+  - enj

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -0,0 +1,241 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/admission/initializer"
+	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
+	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
+	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy"
+	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
+	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
+	apiserverapiv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
+	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/component-base/featuregate"
+)
+
+var configScheme = runtime.NewScheme()
+
+func init() {
+	utilruntime.Must(apiserverapi.AddToScheme(configScheme))
+	utilruntime.Must(apiserverapiv1alpha1.AddToScheme(configScheme))
+	utilruntime.Must(apiserverapiv1.AddToScheme(configScheme))
+}
+
+// AdmissionOptions holds the admission options
+type AdmissionOptions struct {
+	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
+	RecommendedPluginOrder []string
+	// DefaultOffPlugins is a set of plugin names that is disabled by default
+	DefaultOffPlugins sets.String
+
+	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
+	EnablePlugins []string
+	// DisablePlugins indicates plugins to be disabled passed through `--disable-admission-plugins`.
+	DisablePlugins []string
+	// ConfigFile is the file path with admission control configuration.
+	ConfigFile string
+	// Plugins contains all registered plugins.
+	Plugins *admission.Plugins
+	// Decorators is a list of admission decorator to wrap around the admission plugins
+	Decorators admission.Decorators
+}
+
+// NewAdmissionOptions creates a new instance of AdmissionOptions
+// Note:
+//
+//	In addition it calls RegisterAllAdmissionPlugins to register
+//	all generic admission plugins.
+//
+//	Provides the list of RecommendedPluginOrder that holds sane values
+//	that can be used by servers that don't care about admission chain.
+//	Servers that do care can overwrite/append that field after creation.
+func NewAdmissionOptions() *AdmissionOptions {
+	options := &AdmissionOptions{
+		Plugins:    admission.NewPlugins(),
+		Decorators: admission.Decorators{admission.DecoratorFunc(admissionmetrics.WithControllerMetrics)},
+		// This list is mix of mutating admission plugins and validating
+		// admission plugins. The apiserver always runs the validating ones
+		// after all the mutating ones, so their relative order in this list
+		// doesn't matter.
+		RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingadmissionpolicy.PluginName, validatingwebhook.PluginName},
+		DefaultOffPlugins:      sets.NewString(),
+	}
+	server.RegisterAllAdmissionPlugins(options.Plugins)
+	return options
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
+	if a == nil {
+		return
+	}
+
+	fs.StringSliceVar(&a.EnablePlugins, "enable-admission-plugins", a.EnablePlugins, ""+
+		"admission plugins that should be enabled in addition to default enabled ones ("+
+		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
+		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
+		"The order of plugins in this flag does not matter.")
+	fs.StringSliceVar(&a.DisablePlugins, "disable-admission-plugins", a.DisablePlugins, ""+
+		"admission plugins that should be disabled although they are in the default enabled plugins list ("+
+		strings.Join(a.defaultEnabledPluginNames(), ", ")+"). "+
+		"Comma-delimited list of admission plugins: "+strings.Join(a.Plugins.Registered(), ", ")+". "+
+		"The order of plugins in this flag does not matter.")
+	fs.StringVar(&a.ConfigFile, "admission-control-config-file", a.ConfigFile,
+		"File with admission control configuration.")
+}
+
+// ApplyTo adds the admission chain to the server configuration.
+// In case admission plugin names were not provided by a cluster-admin they will be prepared from the recommended/default values.
+// In addition the method lazily initializes a generic plugin that is appended to the list of pluginInitializers
+// note this method uses:
+//
+//	genericconfig.Authorizer
+func (a *AdmissionOptions) ApplyTo(
+	c *server.Config,
+	informers informers.SharedInformerFactory,
+	kubeAPIServerClientConfig *rest.Config,
+	features featuregate.FeatureGate,
+	pluginInitializers ...admission.PluginInitializer,
+) error {
+	if a == nil {
+		return nil
+	}
+
+	// Admission depends on CoreAPI to set SharedInformerFactory and ClientConfig.
+	if informers == nil {
+		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
+	}
+
+	pluginNames := a.enabledPluginNames()
+
+	pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme)
+	if err != nil {
+		return fmt.Errorf("failed to read plugin config: %v", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
+	if err != nil {
+		return err
+	}
+	dynamicClient, err := dynamic.NewForConfig(kubeAPIServerClientConfig)
+	if err != nil {
+		return err
+	}
+	genericInitializer := initializer.New(clientset, dynamicClient, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
+	initializersChain := admission.PluginInitializers{genericInitializer}
+	initializersChain = append(initializersChain, pluginInitializers...)
+
+	admissionChain, err := a.Plugins.NewFromPlugins(pluginNames, pluginsConfigProvider, initializersChain, a.Decorators)
+	if err != nil {
+		return err
+	}
+
+	c.AdmissionControl = admissionmetrics.WithStepMetrics(admissionChain)
+	return nil
+}
+
+// Validate verifies flags passed to AdmissionOptions.
+func (a *AdmissionOptions) Validate() []error {
+	if a == nil {
+		return nil
+	}
+
+	errs := []error{}
+
+	registeredPlugins := sets.NewString(a.Plugins.Registered()...)
+	for _, name := range a.EnablePlugins {
+		if !registeredPlugins.Has(name) {
+			errs = append(errs, fmt.Errorf("enable-admission-plugins plugin %q is unknown", name))
+		}
+	}
+
+	for _, name := range a.DisablePlugins {
+		if !registeredPlugins.Has(name) {
+			errs = append(errs, fmt.Errorf("disable-admission-plugins plugin %q is unknown", name))
+		}
+	}
+
+	enablePlugins := sets.NewString(a.EnablePlugins...)
+	disablePlugins := sets.NewString(a.DisablePlugins...)
+	if len(enablePlugins.Intersection(disablePlugins).List()) > 0 {
+		errs = append(errs, fmt.Errorf("%v in enable-admission-plugins and disable-admission-plugins "+
+			"overlapped", enablePlugins.Intersection(disablePlugins).List()))
+	}
+
+	// Verify RecommendedPluginOrder.
+	recommendPlugins := sets.NewString(a.RecommendedPluginOrder...)
+	intersections := registeredPlugins.Intersection(recommendPlugins)
+	if !intersections.Equal(recommendPlugins) {
+		// Developer error, this should never run in.
+		errs = append(errs, fmt.Errorf("plugins %v in RecommendedPluginOrder are not registered",
+			recommendPlugins.Difference(intersections).List()))
+	}
+	if !intersections.Equal(registeredPlugins) {
+		// Developer error, this should never run in.
+		errs = append(errs, fmt.Errorf("plugins %v registered are not in RecommendedPluginOrder",
+			registeredPlugins.Difference(intersections).List()))
+	}
+
+	return errs
+}
+
+// enabledPluginNames makes use of RecommendedPluginOrder, DefaultOffPlugins,
+// EnablePlugins, DisablePlugins fields
+// to prepare a list of ordered plugin names that are enabled.
+func (a *AdmissionOptions) enabledPluginNames() []string {
+	allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
+	disabledPlugins := sets.NewString(allOffPlugins...)
+	enabledPlugins := sets.NewString(a.EnablePlugins...)
+	disabledPlugins = disabledPlugins.Difference(enabledPlugins)
+
+	orderedPlugins := []string{}
+	for _, plugin := range a.RecommendedPluginOrder {
+		if !disabledPlugins.Has(plugin) {
+			orderedPlugins = append(orderedPlugins, plugin)
+		}
+	}
+
+	return orderedPlugins
+}
+
+// Return names of plugins which are enabled by default
+func (a *AdmissionOptions) defaultEnabledPluginNames() []string {
+	defaultOnPluginNames := []string{}
+	for _, pluginName := range a.RecommendedPluginOrder {
+		if !a.DefaultOffPlugins.Has(pluginName) {
+			defaultOnPluginNames = append(defaultOnPluginNames, pluginName)
+		}
+	}
+
+	return defaultOnPluginNames
+}

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -0,0 +1,115 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/resourceconfig"
+	serverstore "k8s.io/apiserver/pkg/server/storage"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+// APIEnablementOptions contains the options for which resources to turn on and off.
+// Given small aggregated API servers, this option isn't required for "normal" API servers
+type APIEnablementOptions struct {
+	RuntimeConfig cliflag.ConfigurationMap
+}
+
+func NewAPIEnablementOptions() *APIEnablementOptions {
+	return &APIEnablementOptions{
+		RuntimeConfig: make(cliflag.ConfigurationMap),
+	}
+}
+
+// AddFlags adds flags for a specific APIServer to the specified FlagSet
+func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.Var(&s.RuntimeConfig, "runtime-config", ""+
+		"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
+		"v1=true|false for the core API group\n"+
+		"<group>/<version>=true|false for a specific API group and version (e.g. apps/v1=true)\n"+
+		"api/all=true|false controls all API versions\n"+
+		"api/ga=true|false controls all API versions of the form v[0-9]+\n"+
+		"api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+\n"+
+		"api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+\n"+
+		"api/legacy is deprecated, and will be removed in a future version")
+}
+
+// Validate validates RuntimeConfig with a list of registries.
+// Usually this list only has one element, the apiserver registry of the process.
+// But in the advanced (and usually not recommended) case of delegated apiservers there can be more.
+// Validate will filter out the known groups of each registry.
+// If anything is left over after that, an error is returned.
+func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
+	if s == nil {
+		return nil
+	}
+
+	errors := []error{}
+	if s.RuntimeConfig[resourceconfig.APIAll] == "false" && len(s.RuntimeConfig) == 1 {
+		// Do not allow only set api/all=false, in such case apiserver startup has no meaning.
+		return append(errors, fmt.Errorf("invalid key with only %v=false", resourceconfig.APIAll))
+	}
+
+	groups, err := resourceconfig.ParseGroups(s.RuntimeConfig)
+	if err != nil {
+		return append(errors, err)
+	}
+
+	for _, registry := range registries {
+		// filter out known groups
+		groups = unknownGroups(groups, registry)
+	}
+	if len(groups) != 0 {
+		errors = append(errors, fmt.Errorf("unknown api groups %s", strings.Join(groups, ",")))
+	}
+
+	return errors
+}
+
+// ApplyTo override MergedResourceConfig with defaults and registry
+func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
+
+	if s == nil {
+		return nil
+	}
+
+	mergedResourceConfig, err := resourceconfig.MergeAPIResourceConfigs(defaultResourceConfig, s.RuntimeConfig, registry)
+	c.MergedResourceConfig = mergedResourceConfig
+
+	return err
+}
+
+func unknownGroups(groups []string, registry GroupRegistry) []string {
+	unknownGroups := []string{}
+	for _, group := range groups {
+		if !registry.IsGroupRegistered(group) {
+			unknownGroups = append(unknownGroups, group)
+		}
+	}
+	return unknownGroups
+}
+
+// GroupRegistry provides a method to check whether given group is registered.
+type GroupRegistry interface {
+	// IsRegistered returns true if given group is registered.
+	IsGroupRegistered(group string) bool
+}

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -0,0 +1,622 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+	"gopkg.in/natefinch/lumberjack.v2"
+	"k8s.io/klog/v2"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apimachinery/pkg/util/sets"
+	auditinternal "k8s.io/apiserver/pkg/apis/audit"
+	auditv1 "k8s.io/apiserver/pkg/apis/audit/v1"
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/audit/policy"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/webhook"
+	pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
+	pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
+	plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
+	pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
+)
+
+const (
+	// Default configuration values for ModeBatch.
+	defaultBatchBufferSize = 10000 // Buffer up to 10000 events before starting discarding.
+	// These batch parameters are only used by the webhook backend.
+	defaultBatchMaxSize       = 400              // Only send up to 400 events at a time.
+	defaultBatchMaxWait       = 30 * time.Second // Send events at least twice a minute.
+	defaultBatchThrottleQPS   = 10               // Limit the send rate by 10 QPS.
+	defaultBatchThrottleBurst = 15               // Allow up to 15 QPS burst.
+)
+
+func appendBackend(existing, newBackend audit.Backend) audit.Backend {
+	if existing == nil {
+		return newBackend
+	}
+	if newBackend == nil {
+		return existing
+	}
+	return audit.Union(existing, newBackend)
+}
+
+type AuditOptions struct {
+	// Policy configuration file for filtering audit events that are captured.
+	// If unspecified, a default is provided.
+	PolicyFile string
+
+	// Plugin options
+	LogOptions     AuditLogOptions
+	WebhookOptions AuditWebhookOptions
+}
+
+const (
+	// ModeBatch indicates that the audit backend should buffer audit events
+	// internally, sending batch updates either once a certain number of
+	// events have been received or a certain amount of time has passed.
+	ModeBatch = "batch"
+	// ModeBlocking causes the audit backend to block on every attempt to process
+	// a set of events. This causes requests to the API server to wait for the
+	// flush before sending a response.
+	ModeBlocking = "blocking"
+	// ModeBlockingStrict is the same as ModeBlocking, except when there is
+	// a failure during audit logging at RequestReceived stage, the whole
+	// request to apiserver will fail.
+	ModeBlockingStrict = "blocking-strict"
+)
+
+// AllowedModes is the modes known for audit backends.
+var AllowedModes = []string{
+	ModeBatch,
+	ModeBlocking,
+	ModeBlockingStrict,
+}
+
+type AuditBatchOptions struct {
+	// Should the backend asynchronous batch events to the webhook backend or
+	// should the backend block responses?
+	//
+	// Defaults to asynchronous batch events.
+	Mode string
+	// Configuration for batching backend. Only used in batch mode.
+	BatchConfig pluginbuffered.BatchConfig
+}
+
+type AuditTruncateOptions struct {
+	// Whether truncating is enabled or not.
+	Enabled bool
+
+	// Truncating configuration.
+	TruncateConfig plugintruncate.Config
+}
+
+// AuditLogOptions determines the output of the structured audit log by default.
+type AuditLogOptions struct {
+	Path       string
+	MaxAge     int
+	MaxBackups int
+	MaxSize    int
+	Format     string
+	Compress   bool
+
+	BatchOptions    AuditBatchOptions
+	TruncateOptions AuditTruncateOptions
+
+	// API group version used for serializing audit events.
+	GroupVersionString string
+}
+
+// AuditWebhookOptions control the webhook configuration for audit events.
+type AuditWebhookOptions struct {
+	ConfigFile     string
+	InitialBackoff time.Duration
+
+	BatchOptions    AuditBatchOptions
+	TruncateOptions AuditTruncateOptions
+
+	// API group version used for serializing audit events.
+	GroupVersionString string
+}
+
+// AuditDynamicOptions control the configuration of dynamic backends for audit events
+type AuditDynamicOptions struct {
+	// Enabled tells whether the dynamic audit capability is enabled.
+	Enabled bool
+
+	// Configuration for batching backend. This is currently only used as an override
+	// for integration tests
+	BatchConfig *pluginbuffered.BatchConfig
+}
+
+func NewAuditOptions() *AuditOptions {
+	return &AuditOptions{
+		WebhookOptions: AuditWebhookOptions{
+			InitialBackoff: pluginwebhook.DefaultInitialBackoffDelay,
+			BatchOptions: AuditBatchOptions{
+				Mode:        ModeBatch,
+				BatchConfig: defaultWebhookBatchConfig(),
+			},
+			TruncateOptions:    NewAuditTruncateOptions(),
+			GroupVersionString: "audit.k8s.io/v1",
+		},
+		LogOptions: AuditLogOptions{
+			Format: pluginlog.FormatJson,
+			BatchOptions: AuditBatchOptions{
+				Mode:        ModeBlocking,
+				BatchConfig: defaultLogBatchConfig(),
+			},
+			TruncateOptions:    NewAuditTruncateOptions(),
+			GroupVersionString: "audit.k8s.io/v1",
+		},
+	}
+}
+
+func NewAuditTruncateOptions() AuditTruncateOptions {
+	return AuditTruncateOptions{
+		Enabled: false,
+		TruncateConfig: plugintruncate.Config{
+			MaxBatchSize: 10 * 1024 * 1024, // 10MB
+			MaxEventSize: 100 * 1024,       // 100KB
+		},
+	}
+}
+
+// Validate checks invalid config combination
+func (o *AuditOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	var allErrors []error
+	allErrors = append(allErrors, o.LogOptions.Validate()...)
+	allErrors = append(allErrors, o.WebhookOptions.Validate()...)
+
+	return allErrors
+}
+
+func validateBackendMode(pluginName string, mode string) error {
+	for _, m := range AllowedModes {
+		if m == mode {
+			return nil
+		}
+	}
+	return fmt.Errorf("invalid audit %s mode %s, allowed modes are %q", pluginName, mode, strings.Join(AllowedModes, ","))
+}
+
+func validateBackendBatchOptions(pluginName string, options AuditBatchOptions) error {
+	if err := validateBackendMode(pluginName, options.Mode); err != nil {
+		return err
+	}
+	if options.Mode != ModeBatch {
+		// Don't validate the unused options.
+		return nil
+	}
+	config := options.BatchConfig
+	if config.BufferSize <= 0 {
+		return fmt.Errorf("invalid audit batch %s buffer size %v, must be a positive number", pluginName, config.BufferSize)
+	}
+	if config.MaxBatchSize <= 0 {
+		return fmt.Errorf("invalid audit batch %s max batch size %v, must be a positive number", pluginName, config.MaxBatchSize)
+	}
+	if config.ThrottleEnable {
+		if config.ThrottleQPS <= 0 {
+			return fmt.Errorf("invalid audit batch %s throttle QPS %v, must be a positive number", pluginName, config.ThrottleQPS)
+		}
+		if config.ThrottleBurst <= 0 {
+			return fmt.Errorf("invalid audit batch %s throttle burst %v, must be a positive number", pluginName, config.ThrottleBurst)
+		}
+	}
+	return nil
+}
+
+var knownGroupVersions = []schema.GroupVersion{
+	auditv1.SchemeGroupVersion,
+}
+
+func validateGroupVersionString(groupVersion string) error {
+	gv, err := schema.ParseGroupVersion(groupVersion)
+	if err != nil {
+		return err
+	}
+	if !knownGroupVersion(gv) {
+		return fmt.Errorf("invalid group version, allowed versions are %q", knownGroupVersions)
+	}
+	if gv != auditv1.SchemeGroupVersion {
+		klog.Warningf("%q is deprecated and will be removed in a future release, use %q instead", gv, auditv1.SchemeGroupVersion)
+	}
+	return nil
+}
+
+func knownGroupVersion(gv schema.GroupVersion) bool {
+	for _, knownGv := range knownGroupVersions {
+		if gv == knownGv {
+			return true
+		}
+	}
+	return false
+}
+
+func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.PolicyFile, "audit-policy-file", o.PolicyFile,
+		"Path to the file that defines the audit policy configuration.")
+
+	o.LogOptions.AddFlags(fs)
+	o.LogOptions.BatchOptions.AddFlags(pluginlog.PluginName, fs)
+	o.LogOptions.TruncateOptions.AddFlags(pluginlog.PluginName, fs)
+	o.WebhookOptions.AddFlags(fs)
+	o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs)
+	o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs)
+}
+
+func (o *AuditOptions) ApplyTo(
+	c *server.Config,
+) error {
+	if o == nil {
+		return nil
+	}
+	if c == nil {
+		return fmt.Errorf("server config must be non-nil")
+	}
+
+	// 1. Build policy evaluator
+	evaluator, err := o.newPolicyRuleEvaluator()
+	if err != nil {
+		return err
+	}
+
+	// 2. Build log backend
+	var logBackend audit.Backend
+	w, err := o.LogOptions.getWriter()
+	if err != nil {
+		return err
+	}
+	if w != nil {
+		if evaluator == nil {
+			klog.V(2).Info("No audit policy file provided, no events will be recorded for log backend")
+		} else {
+			logBackend = o.LogOptions.newBackend(w)
+		}
+	}
+
+	// 3. Build webhook backend
+	var webhookBackend audit.Backend
+	if o.WebhookOptions.enabled() {
+		if evaluator == nil {
+			klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend")
+		} else {
+			if c.EgressSelector != nil {
+				var egressDialer utilnet.DialFunc
+				egressDialer, err = c.EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext())
+				if err != nil {
+					return err
+				}
+				webhookBackend, err = o.WebhookOptions.newUntruncatedBackend(egressDialer)
+			} else {
+				webhookBackend, err = o.WebhookOptions.newUntruncatedBackend(nil)
+			}
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	groupVersion, err := schema.ParseGroupVersion(o.WebhookOptions.GroupVersionString)
+	if err != nil {
+		return err
+	}
+
+	// 4. Apply dynamic options.
+	var dynamicBackend audit.Backend
+	if webhookBackend != nil {
+		// if only webhook is enabled wrap it in the truncate options
+		dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
+	}
+
+	// 5. Set the policy rule evaluator
+	c.AuditPolicyRuleEvaluator = evaluator
+
+	// 6. Join the log backend with the webhooks
+	c.AuditBackend = appendBackend(logBackend, dynamicBackend)
+
+	if c.AuditBackend != nil {
+		klog.V(2).Infof("Using audit backend: %s", c.AuditBackend)
+	}
+	return nil
+}
+
+func (o *AuditOptions) newPolicyRuleEvaluator() (audit.PolicyRuleEvaluator, error) {
+	if o.PolicyFile == "" {
+		return nil, nil
+	}
+
+	p, err := policy.LoadPolicyFromFile(o.PolicyFile)
+	if err != nil {
+		return nil, fmt.Errorf("loading audit policy file: %v", err)
+	}
+	return policy.NewPolicyRuleEvaluator(p), nil
+}
+
+func (o *AuditBatchOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
+	fs.StringVar(&o.Mode, fmt.Sprintf("audit-%s-mode", pluginName), o.Mode,
+		"Strategy for sending audit events. Blocking indicates sending events should block"+
+			" server responses. Batch causes the backend to buffer and write events"+
+			" asynchronously. Known modes are "+strings.Join(AllowedModes, ",")+".")
+	fs.IntVar(&o.BatchConfig.BufferSize, fmt.Sprintf("audit-%s-batch-buffer-size", pluginName),
+		o.BatchConfig.BufferSize, "The size of the buffer to store events before "+
+			"batching and writing. Only used in batch mode.")
+	fs.IntVar(&o.BatchConfig.MaxBatchSize, fmt.Sprintf("audit-%s-batch-max-size", pluginName),
+		o.BatchConfig.MaxBatchSize, "The maximum size of a batch. Only used in batch mode.")
+	fs.DurationVar(&o.BatchConfig.MaxBatchWait, fmt.Sprintf("audit-%s-batch-max-wait", pluginName),
+		o.BatchConfig.MaxBatchWait, "The amount of time to wait before force writing the "+
+			"batch that hadn't reached the max size. Only used in batch mode.")
+	fs.BoolVar(&o.BatchConfig.ThrottleEnable, fmt.Sprintf("audit-%s-batch-throttle-enable", pluginName),
+		o.BatchConfig.ThrottleEnable, "Whether batching throttling is enabled. Only used in batch mode.")
+	fs.Float32Var(&o.BatchConfig.ThrottleQPS, fmt.Sprintf("audit-%s-batch-throttle-qps", pluginName),
+		o.BatchConfig.ThrottleQPS, "Maximum average number of batches per second. "+
+			"Only used in batch mode.")
+	fs.IntVar(&o.BatchConfig.ThrottleBurst, fmt.Sprintf("audit-%s-batch-throttle-burst", pluginName),
+		o.BatchConfig.ThrottleBurst, "Maximum number of requests sent at the same "+
+			"moment if ThrottleQPS was not utilized before. Only used in batch mode.")
+}
+
+type ignoreErrorsBackend struct {
+	audit.Backend
+}
+
+func (i *ignoreErrorsBackend) ProcessEvents(ev ...*auditinternal.Event) bool {
+	i.Backend.ProcessEvents(ev...)
+	return true
+}
+
+func (i *ignoreErrorsBackend) String() string {
+	return fmt.Sprintf("ignoreErrors<%s>", i.Backend)
+}
+
+func (o *AuditBatchOptions) wrapBackend(delegate audit.Backend) audit.Backend {
+	if o.Mode == ModeBlockingStrict {
+		return delegate
+	}
+	if o.Mode == ModeBlocking {
+		return &ignoreErrorsBackend{Backend: delegate}
+	}
+	return pluginbuffered.NewBackend(delegate, o.BatchConfig)
+}
+
+func (o *AuditTruncateOptions) Validate(pluginName string) error {
+	config := o.TruncateConfig
+	if config.MaxEventSize <= 0 {
+		return fmt.Errorf("invalid audit truncate %s max event size %v, must be a positive number", pluginName, config.MaxEventSize)
+	}
+	if config.MaxBatchSize < config.MaxEventSize {
+		return fmt.Errorf("invalid audit truncate %s max batch size %v, must be greater than "+
+			"max event size (%v)", pluginName, config.MaxBatchSize, config.MaxEventSize)
+	}
+	return nil
+}
+
+func (o *AuditTruncateOptions) AddFlags(pluginName string, fs *pflag.FlagSet) {
+	fs.BoolVar(&o.Enabled, fmt.Sprintf("audit-%s-truncate-enabled", pluginName),
+		o.Enabled, "Whether event and batch truncating is enabled.")
+	fs.Int64Var(&o.TruncateConfig.MaxBatchSize, fmt.Sprintf("audit-%s-truncate-max-batch-size", pluginName),
+		o.TruncateConfig.MaxBatchSize, "Maximum size of the batch sent to the underlying backend. "+
+			"Actual serialized size can be several hundreds of bytes greater. If a batch exceeds this limit, "+
+			"it is split into several batches of smaller size.")
+	fs.Int64Var(&o.TruncateConfig.MaxEventSize, fmt.Sprintf("audit-%s-truncate-max-event-size", pluginName),
+		o.TruncateConfig.MaxEventSize, "Maximum size of the audit event sent to the underlying backend. "+
+			"If the size of an event is greater than this number, first request and response are removed, and "+
+			"if this doesn't reduce the size enough, event is discarded.")
+}
+
+func (o *AuditTruncateOptions) wrapBackend(delegate audit.Backend, gv schema.GroupVersion) audit.Backend {
+	if !o.Enabled {
+		return delegate
+	}
+	return plugintruncate.NewBackend(delegate, o.TruncateConfig, gv)
+}
+
+func (o *AuditLogOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.Path, "audit-log-path", o.Path,
+		"If set, all requests coming to the apiserver will be logged to this file.  '-' means standard out.")
+	fs.IntVar(&o.MaxAge, "audit-log-maxage", o.MaxAge,
+		"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
+	fs.IntVar(&o.MaxBackups, "audit-log-maxbackup", o.MaxBackups,
+		"The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files.")
+	fs.IntVar(&o.MaxSize, "audit-log-maxsize", o.MaxSize,
+		"The maximum size in megabytes of the audit log file before it gets rotated.")
+	fs.StringVar(&o.Format, "audit-log-format", o.Format,
+		"Format of saved audits. \"legacy\" indicates 1-line text format for each event."+
+			" \"json\" indicates structured json format. Known formats are "+
+			strings.Join(pluginlog.AllowedFormats, ",")+".")
+	fs.StringVar(&o.GroupVersionString, "audit-log-version", o.GroupVersionString,
+		"API group and version used for serializing audit events written to log.")
+	fs.BoolVar(&o.Compress, "audit-log-compress", o.Compress, "If set, the rotated log files will be compressed using gzip.")
+}
+
+func (o *AuditLogOptions) Validate() []error {
+	// Check whether the log backend is enabled based on the options.
+	if !o.enabled() {
+		return nil
+	}
+
+	var allErrors []error
+
+	if err := validateBackendBatchOptions(pluginlog.PluginName, o.BatchOptions); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := o.TruncateOptions.Validate(pluginlog.PluginName); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	if err := validateGroupVersionString(o.GroupVersionString); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	// Check log format
+	if !sets.NewString(pluginlog.AllowedFormats...).Has(o.Format) {
+		allErrors = append(allErrors, fmt.Errorf("invalid audit log format %s, allowed formats are %q", o.Format, strings.Join(pluginlog.AllowedFormats, ",")))
+	}
+
+	// Check validities of MaxAge, MaxBackups and MaxSize of log options, if file log backend is enabled.
+	if o.MaxAge < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxage %v can't be a negative number", o.MaxAge))
+	}
+	if o.MaxBackups < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxbackup %v can't be a negative number", o.MaxBackups))
+	}
+	if o.MaxSize < 0 {
+		allErrors = append(allErrors, fmt.Errorf("--audit-log-maxsize %v can't be a negative number", o.MaxSize))
+	}
+
+	return allErrors
+}
+
+// Check whether the log backend is enabled based on the options.
+func (o *AuditLogOptions) enabled() bool {
+	return o != nil && o.Path != ""
+}
+
+func (o *AuditLogOptions) getWriter() (io.Writer, error) {
+	if !o.enabled() {
+		return nil, nil
+	}
+
+	if o.Path == "-" {
+		return os.Stdout, nil
+	}
+
+	if err := o.ensureLogFile(); err != nil {
+		return nil, fmt.Errorf("ensureLogFile: %w", err)
+	}
+
+	return &lumberjack.Logger{
+		Filename:   o.Path,
+		MaxAge:     o.MaxAge,
+		MaxBackups: o.MaxBackups,
+		MaxSize:    o.MaxSize,
+		Compress:   o.Compress,
+	}, nil
+}
+
+func (o *AuditLogOptions) ensureLogFile() error {
+	if err := os.MkdirAll(filepath.Dir(o.Path), 0700); err != nil {
+		return err
+	}
+	mode := os.FileMode(0600)
+	f, err := os.OpenFile(o.Path, os.O_CREATE|os.O_APPEND|os.O_RDWR, mode)
+	if err != nil {
+		return err
+	}
+	return f.Close()
+}
+
+func (o *AuditLogOptions) newBackend(w io.Writer) audit.Backend {
+	groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
+	log := pluginlog.NewBackend(w, o.Format, groupVersion)
+	log = o.BatchOptions.wrapBackend(log)
+	log = o.TruncateOptions.wrapBackend(log, groupVersion)
+	return log
+}
+
+func (o *AuditWebhookOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.ConfigFile, "audit-webhook-config-file", o.ConfigFile,
+		"Path to a kubeconfig formatted file that defines the audit webhook configuration.")
+	fs.DurationVar(&o.InitialBackoff, "audit-webhook-initial-backoff",
+		o.InitialBackoff, "The amount of time to wait before retrying the first failed request.")
+	fs.DurationVar(&o.InitialBackoff, "audit-webhook-batch-initial-backoff",
+		o.InitialBackoff, "The amount of time to wait before retrying the first failed request.")
+	fs.MarkDeprecated("audit-webhook-batch-initial-backoff",
+		"Deprecated, use --audit-webhook-initial-backoff instead.")
+	fs.StringVar(&o.GroupVersionString, "audit-webhook-version", o.GroupVersionString,
+		"API group and version used for serializing audit events written to webhook.")
+}
+
+func (o *AuditWebhookOptions) Validate() []error {
+	if !o.enabled() {
+		return nil
+	}
+
+	var allErrors []error
+	if err := validateBackendBatchOptions(pluginwebhook.PluginName, o.BatchOptions); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := o.TruncateOptions.Validate(pluginwebhook.PluginName); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	if err := validateGroupVersionString(o.GroupVersionString); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	return allErrors
+}
+
+func (o *AuditWebhookOptions) enabled() bool {
+	return o != nil && o.ConfigFile != ""
+}
+
+// newUntruncatedBackend returns a webhook backend without the truncate options applied
+// this is done so that the same trucate backend can wrap both the webhook and dynamic backends
+func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc) (audit.Backend, error) {
+	groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
+	webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, webhook.DefaultRetryBackoffWithInitialDelay(o.InitialBackoff), customDial)
+	if err != nil {
+		return nil, fmt.Errorf("initializing audit webhook: %v", err)
+	}
+	webhook = o.BatchOptions.wrapBackend(webhook)
+	return webhook, nil
+}
+
+// defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend.
+func defaultWebhookBatchConfig() pluginbuffered.BatchConfig {
+	return pluginbuffered.BatchConfig{
+		BufferSize:   defaultBatchBufferSize,
+		MaxBatchSize: defaultBatchMaxSize,
+		MaxBatchWait: defaultBatchMaxWait,
+
+		ThrottleEnable: true,
+		ThrottleQPS:    defaultBatchThrottleQPS,
+		ThrottleBurst:  defaultBatchThrottleBurst,
+
+		AsyncDelegate: true,
+	}
+}
+
+// defaultLogBatchConfig returns the default BatchConfig used by the Log backend.
+func defaultLogBatchConfig() pluginbuffered.BatchConfig {
+	return pluginbuffered.BatchConfig{
+		BufferSize: defaultBatchBufferSize,
+		// Batching is not useful for the log-file backend.
+		// MaxBatchWait ignored.
+		MaxBatchSize:   1,
+		ThrottleEnable: false,
+		// Asynchronous log threads just create lock contention.
+		AsyncDelegate: false,
+	}
+}

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -0,0 +1,447 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+	openapicommon "k8s.io/kube-openapi/pkg/common"
+)
+
+// DefaultAuthWebhookRetryBackoff is the default backoff parameters for
+// both authentication and authorization webhook used by the apiserver.
+func DefaultAuthWebhookRetryBackoff() *wait.Backoff {
+	return &wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   1.5,
+		Jitter:   0.2,
+		Steps:    5,
+	}
+}
+
+type RequestHeaderAuthenticationOptions struct {
+	// ClientCAFile is the root certificate bundle to verify client certificates on incoming requests
+	// before trusting usernames in headers.
+	ClientCAFile string
+
+	UsernameHeaders     []string
+	GroupHeaders        []string
+	ExtraHeaderPrefixes []string
+	AllowedNames        []string
+}
+
+func (s *RequestHeaderAuthenticationOptions) Validate() []error {
+	allErrors := []error{}
+
+	if err := checkForWhiteSpaceOnly("requestheader-username-headers", s.UsernameHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-group-headers", s.GroupHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-extra-headers-prefix", s.ExtraHeaderPrefixes...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-allowed-names", s.AllowedNames...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	return allErrors
+}
+
+func checkForWhiteSpaceOnly(flag string, headerNames ...string) error {
+	for _, headerName := range headerNames {
+		if len(strings.TrimSpace(headerName)) == 0 {
+			return fmt.Errorf("empty value in %q", flag)
+		}
+	}
+
+	return nil
+}
+
+func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.StringSliceVar(&s.UsernameHeaders, "requestheader-username-headers", s.UsernameHeaders, ""+
+		"List of request headers to inspect for usernames. X-Remote-User is common.")
+
+	fs.StringSliceVar(&s.GroupHeaders, "requestheader-group-headers", s.GroupHeaders, ""+
+		"List of request headers to inspect for groups. X-Remote-Group is suggested.")
+
+	fs.StringSliceVar(&s.ExtraHeaderPrefixes, "requestheader-extra-headers-prefix", s.ExtraHeaderPrefixes, ""+
+		"List of request header prefixes to inspect. X-Remote-Extra- is suggested.")
+
+	fs.StringVar(&s.ClientCAFile, "requestheader-client-ca-file", s.ClientCAFile, ""+
+		"Root certificate bundle to use to verify client certificates on incoming requests "+
+		"before trusting usernames in headers specified by --requestheader-username-headers. "+
+		"WARNING: generally do not depend on authorization being already done for incoming requests.")
+
+	fs.StringSliceVar(&s.AllowedNames, "requestheader-allowed-names", s.AllowedNames, ""+
+		"List of client certificate common names to allow to provide usernames in headers "+
+		"specified by --requestheader-username-headers. If empty, any client certificate validated "+
+		"by the authorities in --requestheader-client-ca-file is allowed.")
+}
+
+// ToAuthenticationRequestHeaderConfig returns a RequestHeaderConfig config object for these options
+// if necessary, nil otherwise.
+func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() (*authenticatorfactory.RequestHeaderConfig, error) {
+	if len(s.ClientCAFile) == 0 {
+		return nil, nil
+	}
+
+	caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &authenticatorfactory.RequestHeaderConfig{
+		UsernameHeaders:     headerrequest.StaticStringSlice(s.UsernameHeaders),
+		GroupHeaders:        headerrequest.StaticStringSlice(s.GroupHeaders),
+		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
+		CAContentProvider:   caBundleProvider,
+		AllowedClientNames:  headerrequest.StaticStringSlice(s.AllowedNames),
+	}, nil
+}
+
+// ClientCertAuthenticationOptions provides different options for client cert auth. You should use `GetClientVerifyOptionFn` to
+// get the verify options for your authenticator.
+type ClientCertAuthenticationOptions struct {
+	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
+	ClientCA string
+
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// Generally this is the CA bundle file used to authenticate client certificates
+	// If non-nil, this takes priority over the ClientCA file.
+	CAContentProvider dynamiccertificates.CAContentProvider
+}
+
+// GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
+func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
+	if s.CAContentProvider != nil {
+		return s.CAContentProvider, nil
+	}
+
+	if len(s.ClientCA) == 0 {
+		return nil, nil
+	}
+
+	return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
+}
+
+func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&s.ClientCA, "client-ca-file", s.ClientCA, ""+
+		"If set, any request presenting a client certificate signed by one of "+
+		"the authorities in the client-ca-file is authenticated with an identity "+
+		"corresponding to the CommonName of the client certificate.")
+}
+
+// DelegatingAuthenticationOptions provides an easy way for composing API servers to delegate their authentication to
+// the root kube API server.  The API federator will act as
+// a front proxy and direction connections will be able to delegate to the core kube API server
+type DelegatingAuthenticationOptions struct {
+	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
+	// TokenAccessReview.authentication.k8s.io endpoint for checking tokens.
+	RemoteKubeConfigFile string
+	// RemoteKubeConfigFileOptional is specifying whether not specifying the kubeconfig or
+	// a missing in-cluster config will be fatal.
+	RemoteKubeConfigFileOptional bool
+
+	// CacheTTL is the length of time that a token authentication answer will be cached.
+	CacheTTL time.Duration
+
+	ClientCert    ClientCertAuthenticationOptions
+	RequestHeader RequestHeaderAuthenticationOptions
+
+	// SkipInClusterLookup indicates missing authentication configuration should not be retrieved from the cluster configmap
+	SkipInClusterLookup bool
+
+	// TolerateInClusterLookupFailure indicates failures to look up authentication configuration from the cluster configmap should not be fatal.
+	// Setting this can result in an authenticator that will reject all requests.
+	TolerateInClusterLookupFailure bool
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
+	// TokenRequestTimeout specifies a time limit for requests made by the authorization webhook client.
+	// The default value is set to 10 seconds.
+	TokenRequestTimeout time.Duration
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
+
+	// DisableAnonymous gives user an option to disable Anonymous authentication.
+	DisableAnonymous bool
+}
+
+func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
+	return &DelegatingAuthenticationOptions{
+		// very low for responsiveness, but high enough to handle storms
+		CacheTTL:   10 * time.Second,
+		ClientCert: ClientCertAuthenticationOptions{},
+		RequestHeader: RequestHeaderAuthenticationOptions{
+			UsernameHeaders:     []string{"x-remote-user"},
+			GroupHeaders:        []string{"x-remote-group"},
+			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
+		},
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
+		TokenRequestTimeout: 10 * time.Second,
+	}
+}
+
+// WithCustomRetryBackoff sets the custom backoff parameters for the authentication webhook retry logic.
+func (s *DelegatingAuthenticationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
+// WithRequestTimeout sets the given timeout for requests made by the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithRequestTimeout(timeout time.Duration) {
+	s.TokenRequestTimeout = timeout
+}
+
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
+func (s *DelegatingAuthenticationOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	allErrors = append(allErrors, s.RequestHeader.Validate()...)
+
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
+	return allErrors
+}
+
+func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	var optionalKubeConfigSentence string
+	if s.RemoteKubeConfigFileOptional {
+		optionalKubeConfigSentence = " This is optional. If empty, all token requests are considered to be anonymous and no client CA is looked up in the cluster."
+	}
+	fs.StringVar(&s.RemoteKubeConfigFile, "authentication-kubeconfig", s.RemoteKubeConfigFile, ""+
+		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+
+		"tokenreviews.authentication.k8s.io."+optionalKubeConfigSentence)
+
+	fs.DurationVar(&s.CacheTTL, "authentication-token-webhook-cache-ttl", s.CacheTTL,
+		"The duration to cache responses from the webhook token authenticator.")
+
+	s.ClientCert.AddFlags(fs)
+	s.RequestHeader.AddFlags(fs)
+
+	fs.BoolVar(&s.SkipInClusterLookup, "authentication-skip-lookup", s.SkipInClusterLookup, ""+
+		"If false, the authentication-kubeconfig will be used to lookup missing authentication "+
+		"configuration from the cluster.")
+	fs.BoolVar(&s.TolerateInClusterLookupFailure, "authentication-tolerate-lookup-failure", s.TolerateInClusterLookupFailure, ""+
+		"If true, failures to look up missing authentication configuration from the cluster are not considered fatal. "+
+		"Note that this can result in authentication that treats all requests as anonymous.")
+}
+
+func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
+	if s == nil {
+		authenticationInfo.Authenticator = nil
+		return nil
+	}
+
+	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
+		Anonymous:                !s.DisableAnonymous,
+		CacheTTL:                 s.CacheTTL,
+		WebhookRetryBackoff:      s.WebhookRetryBackoff,
+		TokenAccessReviewTimeout: s.TokenRequestTimeout,
+	}
+
+	client, err := s.getClient()
+	if err != nil {
+		return fmt.Errorf("failed to get delegated authentication kubeconfig: %v", err)
+	}
+
+	// configure token review
+	if client != nil {
+		cfg.TokenAccessReviewClient = client.AuthenticationV1()
+	}
+
+	// get the clientCA information
+	clientCASpecified := s.ClientCert != ClientCertAuthenticationOptions{}
+	var clientCAProvider dynamiccertificates.CAContentProvider
+	if clientCASpecified {
+		clientCAProvider, err = s.ClientCert.GetClientCAContentProvider()
+		if err != nil {
+			return fmt.Errorf("unable to load client CA provider: %v", err)
+		}
+		cfg.ClientCertificateCAContentProvider = clientCAProvider
+		if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to assign client CA provider: %v", err)
+		}
+
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			clientCAProvider, err = dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "client-ca-file", client)
+			if err != nil {
+				return fmt.Errorf("unable to load configmap based client CA file: %v", err)
+			}
+			cfg.ClientCertificateCAContentProvider = clientCAProvider
+			if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+				return fmt.Errorf("unable to assign configmap based client CA file: %v", err)
+			}
+
+		}
+	}
+
+	requestHeaderCAFileSpecified := len(s.RequestHeader.ClientCAFile) > 0
+	var requestHeaderConfig *authenticatorfactory.RequestHeaderConfig
+	if requestHeaderCAFileSpecified {
+		requestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		if err != nil {
+			return fmt.Errorf("unable to create request header authentication config: %v", err)
+		}
+
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			requestHeaderConfig, err = s.createRequestHeaderConfig(client)
+			if err != nil {
+				if s.TolerateInClusterLookupFailure {
+					klog.Warningf("Error looking up in-cluster authentication configuration: %v", err)
+					klog.Warning("Continuing without authentication configuration. This may treat all requests as anonymous.")
+					klog.Warning("To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false")
+				} else {
+					return fmt.Errorf("unable to load configmap based request-header-client-ca-file: %v", err)
+				}
+			}
+		}
+	}
+	if requestHeaderConfig != nil {
+		cfg.RequestHeaderConfig = requestHeaderConfig
+		if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to load request-header-client-ca-file: %v", err)
+		}
+	}
+
+	// create authenticator
+	authenticator, securityDefinitions, err := cfg.New()
+	if err != nil {
+		return err
+	}
+	authenticationInfo.Authenticator = authenticator
+	if openAPIConfig != nil {
+		openAPIConfig.SecurityDefinitions = securityDefinitions
+	}
+
+	return nil
+}
+
+const (
+	authenticationConfigMapNamespace = metav1.NamespaceSystem
+	// authenticationConfigMapName is the name of ConfigMap in the kube-system namespace holding the root certificate
+	// bundle to use to verify client certificates on incoming requests before trusting usernames in headers specified
+	// by --requestheader-username-headers. This is created in the cluster by the kube-apiserver.
+	// "WARNING: generally do not depend on authorization being already done for incoming requests.")
+	authenticationConfigMapName = "extension-apiserver-authentication"
+)
+
+func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
+	dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
+	}
+
+	//  look up authentication configuration in the cluster and in case of an err defer to authentication-tolerate-lookup-failure flag
+	//  We are passing the context to ProxyCerts.RunOnce as it needs to implement RunOnce(ctx) however the
+	//  context is not used at all. So passing a empty context shouldn't be a problem
+	ctx := context.TODO()
+	if err := dynamicRequestHeaderProvider.RunOnce(ctx); err != nil {
+		return nil, err
+	}
+
+	return &authenticatorfactory.RequestHeaderConfig{
+		CAContentProvider:   dynamicRequestHeaderProvider,
+		UsernameHeaders:     headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)),
+		GroupHeaders:        headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)),
+		ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)),
+		AllowedClientNames:  headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)),
+	}, nil
+}
+
+// getClient returns a Kubernetes clientset. If s.RemoteKubeConfigFileOptional is true, nil will be returned
+// if no kubeconfig is specified by the user and the in-cluster config is not found.
+func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, error) {
+	var clientConfig *rest.Config
+	var err error
+	if len(s.RemoteKubeConfigFile) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		clientConfig, err = loader.ClientConfig()
+	} else {
+		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
+		// use this path. If it is optional, ignore errors.
+		clientConfig, err = rest.InClusterConfig()
+		if err != nil && s.RemoteKubeConfigFileOptional {
+			if err != rest.ErrNotInCluster {
+				klog.Warningf("failed to read in-cluster kubeconfig for delegated authentication: %v", err)
+			}
+			return nil, nil
+		}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get delegated authentication kubeconfig: %v", err)
+	}
+
+	// set high qps/burst limits since this will effectively limit API server responsiveness
+	clientConfig.QPS = 200
+	clientConfig.Burst = 400
+	// do not set a timeout on the http client, instead use context for cancellation
+	// if multiple timeouts were set, the request will pick the smaller timeout to be applied, leaving other useless.
+	//
+	// see https://github.com/golang/go/blob/a937729c2c2f6950a32bc5cd0f5b88700882f078/src/net/http/client.go#L364
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
+
+	return kubernetes.NewForConfig(clientConfig)
+}

vendor/k8s.io/apiserver/pkg/server/options/authentication_dynamic_request_header.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ dynamiccertificates.ControllerRunner = &DynamicRequestHeaderController{}
+var _ dynamiccertificates.CAContentProvider = &DynamicRequestHeaderController{}
+
+var _ headerrequest.RequestHeaderAuthRequestProvider = &DynamicRequestHeaderController{}
+
+// DynamicRequestHeaderController combines DynamicCAFromConfigMapController and RequestHeaderAuthRequestController
+// into one controller for dynamically filling RequestHeaderConfig struct
+type DynamicRequestHeaderController struct {
+	*dynamiccertificates.ConfigMapCAController
+	*headerrequest.RequestHeaderAuthRequestController
+}
+
+// newDynamicRequestHeaderController creates a new controller that implements DynamicRequestHeaderController
+func newDynamicRequestHeaderController(client kubernetes.Interface) (*DynamicRequestHeaderController, error) {
+	requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+		"client-ca",
+		authenticationConfigMapNamespace,
+		authenticationConfigMapName,
+		"requestheader-client-ca-file",
+		client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err)
+	}
+
+	requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController(
+		authenticationConfigMapName,
+		authenticationConfigMapNamespace,
+		client,
+		"requestheader-username-headers",
+		"requestheader-group-headers",
+		"requestheader-extra-headers-prefix",
+		"requestheader-allowed-names",
+	)
+	return &DynamicRequestHeaderController{
+		ConfigMapCAController:              requestHeaderCAController,
+		RequestHeaderAuthRequestController: requestHeaderAuthRequestController,
+	}, nil
+}
+
+func (c *DynamicRequestHeaderController) RunOnce(ctx context.Context) error {
+	errs := []error{}
+	errs = append(errs, c.ConfigMapCAController.RunOnce(ctx))
+	errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce(ctx))
+	return errors.NewAggregate(errs)
+}
+
+func (c *DynamicRequestHeaderController) Run(ctx context.Context, workers int) {
+	go c.ConfigMapCAController.Run(ctx, workers)
+	go c.RequestHeaderAuthRequestController.Run(ctx, workers)
+	<-ctx.Done()
+}

vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -0,0 +1,244 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
+	"k8s.io/apiserver/pkg/authorization/path"
+	"k8s.io/apiserver/pkg/authorization/union"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/transport"
+	"k8s.io/klog/v2"
+)
+
+// DelegatingAuthorizationOptions provides an easy way for composing API servers to delegate their authorization to
+// the root kube API server.
+// WARNING: never assume that every authenticated incoming request already does authorization.
+//
+//	The aggregator in the kube API server does this today, but this behaviour is not
+//	guaranteed in the future.
+type DelegatingAuthorizationOptions struct {
+	// RemoteKubeConfigFile is the file to use to connect to a "normal" kube API server which hosts the
+	// SubjectAccessReview.authorization.k8s.io endpoint for checking tokens.
+	RemoteKubeConfigFile string
+	// RemoteKubeConfigFileOptional is specifying whether not specifying the kubeconfig or
+	// a missing in-cluster config will be fatal.
+	RemoteKubeConfigFileOptional bool
+
+	// AllowCacheTTL is the length of time that a successful authorization response will be cached
+	AllowCacheTTL time.Duration
+
+	// DenyCacheTTL is the length of time that an unsuccessful authorization response will be cached.
+	// You generally want more responsive, "deny, try again" flows.
+	DenyCacheTTL time.Duration
+
+	// AlwaysAllowPaths are HTTP paths which are excluded from authorization. They can be plain
+	// paths or end in * in which case prefix-match is applied. A leading / is optional.
+	AlwaysAllowPaths []string
+
+	// AlwaysAllowGroups are groups which are allowed to take any actions.  In kube, this is system:masters.
+	AlwaysAllowGroups []string
+
+	// ClientTimeout specifies a time limit for requests made by SubjectAccessReviews client.
+	// The default value is set to 10 seconds.
+	ClientTimeout time.Duration
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
+	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+	CustomRoundTripperFn transport.WrapperFunc
+}
+
+func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions {
+	return &DelegatingAuthorizationOptions{
+		// very low for responsiveness, but high enough to handle storms
+		AllowCacheTTL:       10 * time.Second,
+		DenyCacheTTL:        10 * time.Second,
+		ClientTimeout:       10 * time.Second,
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
+		// This allows the kubelet to always get health and readiness without causing an authorization check.
+		// This field can be cleared by callers if they don't want this behavior.
+		AlwaysAllowPaths: []string{"/healthz", "/readyz", "/livez"},
+		// In an authorization call delegated to a kube-apiserver (the expected common-case), system:masters has full
+		// authority in a hard-coded authorizer.  This means that our default can reasonably be to skip an authorization
+		// check for system:masters.
+		// This field can be cleared by callers if they don't want this behavior.
+		AlwaysAllowGroups: []string{"system:masters"},
+	}
+}
+
+// WithAlwaysAllowGroups appends the list of paths to AlwaysAllowGroups
+func (s *DelegatingAuthorizationOptions) WithAlwaysAllowGroups(groups ...string) *DelegatingAuthorizationOptions {
+	s.AlwaysAllowGroups = append(s.AlwaysAllowGroups, groups...)
+	return s
+}
+
+// WithAlwaysAllowPaths appends the list of paths to AlwaysAllowPaths
+func (s *DelegatingAuthorizationOptions) WithAlwaysAllowPaths(paths ...string) *DelegatingAuthorizationOptions {
+	s.AlwaysAllowPaths = append(s.AlwaysAllowPaths, paths...)
+	return s
+}
+
+// WithClientTimeout sets the given timeout for SAR client used by this authorizer
+func (s *DelegatingAuthorizationOptions) WithClientTimeout(timeout time.Duration) {
+	s.ClientTimeout = timeout
+}
+
+// WithCustomRetryBackoff sets the custom backoff parameters for the authorization webhook retry logic.
+func (s *DelegatingAuthorizationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
+// WithCustomRoundTripper allows for specifying a middleware function for custom HTTP behaviour for the authorization webhook client.
+func (s *DelegatingAuthorizationOptions) WithCustomRoundTripper(rt transport.WrapperFunc) {
+	s.CustomRoundTripperFn = rt
+}
+
+func (s *DelegatingAuthorizationOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
+	return allErrors
+}
+
+func (s *DelegatingAuthorizationOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	var optionalKubeConfigSentence string
+	if s.RemoteKubeConfigFileOptional {
+		optionalKubeConfigSentence = " This is optional. If empty, all requests not skipped by authorization are forbidden."
+	}
+	fs.StringVar(&s.RemoteKubeConfigFile, "authorization-kubeconfig", s.RemoteKubeConfigFile,
+		"kubeconfig file pointing at the 'core' kubernetes server with enough rights to create "+
+			"subjectaccessreviews.authorization.k8s.io."+optionalKubeConfigSentence)
+
+	fs.DurationVar(&s.AllowCacheTTL, "authorization-webhook-cache-authorized-ttl",
+		s.AllowCacheTTL,
+		"The duration to cache 'authorized' responses from the webhook authorizer.")
+
+	fs.DurationVar(&s.DenyCacheTTL,
+		"authorization-webhook-cache-unauthorized-ttl", s.DenyCacheTTL,
+		"The duration to cache 'unauthorized' responses from the webhook authorizer.")
+
+	fs.StringSliceVar(&s.AlwaysAllowPaths, "authorization-always-allow-paths", s.AlwaysAllowPaths,
+		"A list of HTTP paths to skip during authorization, i.e. these are authorized without "+
+			"contacting the 'core' kubernetes server.")
+}
+
+func (s *DelegatingAuthorizationOptions) ApplyTo(c *server.AuthorizationInfo) error {
+	if s == nil {
+		c.Authorizer = authorizerfactory.NewAlwaysAllowAuthorizer()
+		return nil
+	}
+
+	client, err := s.getClient()
+	if err != nil {
+		return err
+	}
+
+	c.Authorizer, err = s.toAuthorizer(client)
+	return err
+}
+
+func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interface) (authorizer.Authorizer, error) {
+	var authorizers []authorizer.Authorizer
+
+	if len(s.AlwaysAllowGroups) > 0 {
+		authorizers = append(authorizers, authorizerfactory.NewPrivilegedGroups(s.AlwaysAllowGroups...))
+	}
+
+	if len(s.AlwaysAllowPaths) > 0 {
+		a, err := path.NewAuthorizer(s.AlwaysAllowPaths)
+		if err != nil {
+			return nil, err
+		}
+		authorizers = append(authorizers, a)
+	}
+
+	if client == nil {
+		klog.Warning("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.")
+	} else {
+		cfg := authorizerfactory.DelegatingAuthorizerConfig{
+			SubjectAccessReviewClient: client.AuthorizationV1(),
+			AllowCacheTTL:             s.AllowCacheTTL,
+			DenyCacheTTL:              s.DenyCacheTTL,
+			WebhookRetryBackoff:       s.WebhookRetryBackoff,
+		}
+		delegatedAuthorizer, err := cfg.New()
+		if err != nil {
+			return nil, err
+		}
+		authorizers = append(authorizers, delegatedAuthorizer)
+	}
+
+	return union.New(authorizers...), nil
+}
+
+func (s *DelegatingAuthorizationOptions) getClient() (kubernetes.Interface, error) {
+	var clientConfig *rest.Config
+	var err error
+	if len(s.RemoteKubeConfigFile) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: s.RemoteKubeConfigFile}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+
+		clientConfig, err = loader.ClientConfig()
+	} else {
+		// without the remote kubeconfig file, try to use the in-cluster config.  Most addon API servers will
+		// use this path. If it is optional, ignore errors.
+		clientConfig, err = rest.InClusterConfig()
+		if err != nil && s.RemoteKubeConfigFileOptional {
+			if err != rest.ErrNotInCluster {
+				klog.Warningf("failed to read in-cluster kubeconfig for delegated authorization: %v", err)
+			}
+			return nil, nil
+		}
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to get delegated authorization kubeconfig: %v", err)
+	}
+
+	// set high qps/burst limits since this will effectively limit API server responsiveness
+	clientConfig.QPS = 200
+	clientConfig.Burst = 400
+	clientConfig.Timeout = s.ClientTimeout
+	if s.CustomRoundTripperFn != nil {
+		clientConfig.Wrap(s.CustomRoundTripperFn)
+	}
+
+	return kubernetes.NewForConfig(clientConfig)
+}

vendor/k8s.io/apiserver/pkg/server/options/coreapi.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/util/feature"
+	clientgoinformers "k8s.io/client-go/informers"
+	clientgoclientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	tracing "k8s.io/component-base/tracing"
+)
+
+// CoreAPIOptions contains options to configure the connection to a core API Kubernetes apiserver.
+type CoreAPIOptions struct {
+	// CoreAPIKubeconfigPath is a filename for a kubeconfig file to contact the core API server with.
+	// If it is not set, the in cluster config is used.
+	CoreAPIKubeconfigPath string
+}
+
+func NewCoreAPIOptions() *CoreAPIOptions {
+	return &CoreAPIOptions{}
+}
+
+func (o *CoreAPIOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.CoreAPIKubeconfigPath, "kubeconfig", o.CoreAPIKubeconfigPath,
+		"kubeconfig file pointing at the 'core' kubernetes server.")
+}
+
+func (o *CoreAPIOptions) ApplyTo(config *server.RecommendedConfig) error {
+	if o == nil {
+		return nil
+	}
+
+	// create shared informer for Kubernetes APIs
+	var kubeconfig *rest.Config
+	var err error
+	if len(o.CoreAPIKubeconfigPath) > 0 {
+		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.CoreAPIKubeconfigPath}
+		loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, &clientcmd.ConfigOverrides{})
+		kubeconfig, err = loader.ClientConfig()
+		if err != nil {
+			return fmt.Errorf("failed to load kubeconfig at %q: %v", o.CoreAPIKubeconfigPath, err)
+		}
+	} else {
+		kubeconfig, err = rest.InClusterConfig()
+		if err != nil {
+			return err
+		}
+	}
+	if feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		kubeconfig.Wrap(tracing.WrapperFor(config.TracerProvider))
+	}
+	clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeconfig)
+	if err != nil {
+		return fmt.Errorf("failed to create Kubernetes clientset: %v", err)
+	}
+	config.ClientConfig = kubeconfig
+	config.SharedInformerFactory = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
+
+	return nil
+}
+
+func (o *CoreAPIOptions) Validate() []error {
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -0,0 +1,126 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apiserver/pkg/server"
+)
+
+// DeprecatedInsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
+// No one should be using these anymore.
+// DEPRECATED: all insecure serving options are removed in a future version
+type DeprecatedInsecureServingOptions struct {
+	BindAddress net.IP
+	BindPort    int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
+
+	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
+	// It defaults to options.CreateListener.
+	ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
+}
+
+// Validate ensures that the insecure port values within the range of the port.
+func (s *DeprecatedInsecureServingOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	errors := []error{}
+
+	if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
+	}
+
+	return errors
+}
+
+// AddFlags adds flags related to insecure serving to the specified FlagSet.
+func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
+		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).")
+	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
+	fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
+	fs.Lookup("insecure-bind-address").Hidden = false
+
+	fs.IntVar(&s.BindPort, "insecure-port", s.BindPort, ""+
+		"The port on which to serve unsecured, unauthenticated access.")
+	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
+	fs.MarkDeprecated("insecure-port", "This flag will be removed in a future version.")
+	fs.Lookup("insecure-port").Hidden = false
+}
+
+// AddUnqualifiedFlags adds flags related to insecure serving without the --insecure prefix to the specified FlagSet.
+func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
+		"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening in all interfaces and IP families).")
+	fs.MarkDeprecated("address", "see --bind-address instead.")
+	fs.Lookup("address").Hidden = false
+
+	fs.IntVar(&s.BindPort, "port", s.BindPort, "The port on which to serve unsecured, unauthenticated access. Set to 0 to disable.")
+	fs.MarkDeprecated("port", "see --secure-port instead.")
+	fs.Lookup("port").Hidden = false
+}
+
+// ApplyTo adds DeprecatedInsecureServingOptions to the insecureserverinfo and kube-controller manager configuration.
+// Note: the double pointer allows to set the *DeprecatedInsecureServingInfo to nil without referencing the struct hosting this pointer.
+func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecureServingInfo) error {
+	if s == nil {
+		return nil
+	}
+	if s.BindPort <= 0 {
+		return nil
+	}
+
+	if s.Listener == nil {
+		var err error
+		listen := CreateListener
+		if s.ListenFunc != nil {
+			listen = s.ListenFunc
+		}
+		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
+		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	}
+
+	*c = &server.DeprecatedInsecureServingInfo{
+		Listener: s.Listener,
+	}
+
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/options/doc.go

@@ -0,0 +1,21 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// package options is the public flags and options used by a generic api
+// server. It takes a minimal set of dependencies and does not reference
+// implementations, in order to ensure it may be reused by multiple components
+// (such as CLI commands that wish to generate or validate config).
+package options // import "k8s.io/apiserver/pkg/server/options"

vendor/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+	"k8s.io/utils/path"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+)
+
+// EgressSelectorOptions holds the api server egress selector options.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/1281-network-proxy/README.md
+type EgressSelectorOptions struct {
+	// ConfigFile is the file path with api-server egress selector configuration.
+	ConfigFile string
+}
+
+// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions
+//
+// The option is to point to a configuration file for egress/konnectivity.
+// This determines which types of requests use egress/konnectivity and how they use it.
+// If empty the API Server will attempt to connect directly using the network.
+func NewEgressSelectorOptions() *EgressSelectorOptions {
+	return &EgressSelectorOptions{}
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile,
+		"File with apiserver egress selector configuration.")
+}
+
+// ApplyTo adds the egress selector settings to the server configuration.
+// In case egress selector settings were not provided by a cluster-admin
+// they will be prepared from the recommended/default/no-op values.
+func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read egress selector config: %v", err)
+	}
+	errs := egressselector.ValidateEgressSelectorConfiguration(npConfig)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate())
+	}
+
+	cs, err := egressselector.NewEgressSelector(npConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err)
+	}
+	c.EgressSelector = cs
+	return nil
+}
+
+// Validate verifies flags passed to EgressSelectorOptions.
+func (o *EgressSelectorOptions) Validate() []error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+
+	errs := []error{}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists || err != nil {
+		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
+	}
+
+	return errs
+}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+  - sig-auth-encryption-at-rest-approvers
+reviewers:
+  - sig-auth-encryption-at-rest-reviewers
+labels:
+  - sig/auth

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -0,0 +1,730 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package encryptionconfig
+
+import (
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	"k8s.io/apiserver/pkg/apis/config/validation"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/storage/value"
+	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+	envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
+
+const (
+	aesCBCTransformerPrefixV1    = "k8s:enc:aescbc:v1:"
+	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
+	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
+	kmsTransformerPrefixV1       = "k8s:enc:kms:v1:"
+	kmsTransformerPrefixV2       = "k8s:enc:kms:v2:"
+	kmsPluginHealthzNegativeTTL  = 3 * time.Second
+	kmsPluginHealthzPositiveTTL  = 20 * time.Second
+	kmsAPIVersionV1              = "v1"
+	kmsAPIVersionV2              = "v2"
+	kmsReloadHealthCheckName     = "kms-providers"
+)
+
+type kmsPluginHealthzResponse struct {
+	err      error
+	received time.Time
+}
+
+type kmsPluginProbe struct {
+	name         string
+	ttl          time.Duration
+	service      envelope.Service
+	lastResponse *kmsPluginHealthzResponse
+	l            *sync.Mutex
+}
+
+type kmsv2PluginProbe struct {
+	name         string
+	ttl          time.Duration
+	service      envelopekmsv2.Service
+	lastResponse *kmsPluginHealthzResponse
+	l            *sync.Mutex
+}
+
+type kmsHealthChecker []healthz.HealthChecker
+
+func (k kmsHealthChecker) Name() string {
+	return kmsReloadHealthCheckName
+}
+
+func (k kmsHealthChecker) Check(req *http.Request) error {
+	var errs []error
+
+	for i := range k {
+		checker := k[i]
+		if err := checker.Check(req); err != nil {
+			errs = append(errs, fmt.Errorf("%s: %w", checker.Name(), err))
+		}
+	}
+
+	return utilerrors.Reduce(utilerrors.NewAggregate(errs))
+}
+
+func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
+		return h.check()
+	})
+}
+
+func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
+		return h.check(r.Context())
+	})
+}
+
+// EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
+type EncryptionConfiguration struct {
+	// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
+	Transformers map[schema.GroupResource]value.Transformer
+
+	// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
+	HealthChecks []healthz.HealthChecker
+
+	// EncryptionFileContentHash is the hash of the encryption config file.
+	EncryptionFileContentHash string
+
+	// KMSCloseGracePeriod is the duration we will wait before closing old transformers.
+	// We wait for any in-flight requests to finish by using the duration which is longer than their timeout.
+	KMSCloseGracePeriod time.Duration
+}
+
+// LoadEncryptionConfig parses and validates the encryption config specified by filepath.
+// It may launch multiple go routines whose lifecycle is controlled by stopCh.
+// If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
+func LoadEncryptionConfig(filepath string, reload bool, stopCh <-chan struct{}) (*EncryptionConfiguration, error) {
+	config, contentHash, err := loadConfig(filepath, reload)
+	if err != nil {
+		return nil, fmt.Errorf("error while parsing file: %w", err)
+	}
+
+	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(config, stopCh)
+	if err != nil {
+		return nil, fmt.Errorf("error while building transformers: %w", err)
+	}
+
+	if reload || (kmsUsed.v2Used && !kmsUsed.v1Used) {
+		kmsHealthChecks = []healthz.HealthChecker{kmsHealthChecker(kmsHealthChecks)}
+	}
+
+	// KMSTimeout is the duration we will wait before closing old transformers.
+	// The way we calculate is as follows:
+	// 1. Sum all timeouts across all KMS plugins. (check kmsPrefixTransformer for differences between v1 and v2)
+	// 2. Multiply that by 2 (to allow for some buffer)
+	// The reason we sum all timeout is because kmsHealthChecker() will run all health checks serially
+	return &EncryptionConfiguration{
+		Transformers:              transformers,
+		HealthChecks:              kmsHealthChecks,
+		EncryptionFileContentHash: contentHash,
+		KMSCloseGracePeriod:       2 * kmsUsed.kmsTimeoutSum,
+	}, err
+}
+
+func getTransformerOverridesAndKMSPluginHealthzCheckers(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
+	var kmsHealthChecks []healthz.HealthChecker
+	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(config, stopCh)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	for i := range probes {
+		probe := probes[i]
+		kmsHealthChecks = append(kmsHealthChecks, probe.toHealthzCheck(i))
+	}
+
+	return transformers, kmsHealthChecks, kmsUsed, nil
+}
+
+type healthChecker interface {
+	toHealthzCheck(idx int) healthz.HealthChecker
+}
+
+func getTransformerOverridesAndKMSPluginProbes(config *apiserverconfig.EncryptionConfiguration, stopCh <-chan struct{}) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
+	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
+	var probes []healthChecker
+	var kmsUsed kmsState
+
+	// For each entry in the configuration
+	for _, resourceConfig := range config.Resources {
+		resourceConfig := resourceConfig
+
+		transformers, p, used, err := prefixTransformersAndProbes(resourceConfig, stopCh)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
+		kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
+
+		kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
+
+		// For each resource, create a list of providers to use
+		for _, resource := range resourceConfig.Resources {
+			resource := resource
+			gr := schema.ParseGroupResource(resource)
+			resourceToPrefixTransformer[gr] = append(
+				resourceToPrefixTransformer[gr], transformers...)
+		}
+
+		probes = append(probes, p...)
+	}
+
+	transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
+	for gr, transList := range resourceToPrefixTransformer {
+		gr := gr
+		transList := transList
+		transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
+	}
+
+	return transformers, probes, &kmsUsed, nil
+}
+
+// check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
+func (h *kmsPluginProbe) check() error {
+	h.l.Lock()
+	defer h.l.Unlock()
+
+	if (time.Since(h.lastResponse.received)) < h.ttl {
+		return h.lastResponse.err
+	}
+
+	p, err := h.service.Encrypt([]byte("ping"))
+	if err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
+	}
+
+	if _, err := h.service.Decrypt(p); err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %w", h.name, err)
+	}
+
+	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	h.ttl = kmsPluginHealthzPositiveTTL
+	return nil
+}
+
+// check gets the healthz status of the KMSv2-Plugin using the Status() method.
+func (h *kmsv2PluginProbe) check(ctx context.Context) error {
+	h.l.Lock()
+	defer h.l.Unlock()
+
+	if (time.Since(h.lastResponse.received)) < h.ttl {
+		return h.lastResponse.err
+	}
+
+	p, err := h.service.Status(ctx)
+	if err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return fmt.Errorf("failed to perform status section of the healthz check for KMS Provider %s, error: %w", h.name, err)
+	}
+
+	if err := isKMSv2ProviderHealthy(h.name, p); err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		h.ttl = kmsPluginHealthzNegativeTTL
+		return err
+	}
+
+	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	h.ttl = kmsPluginHealthzPositiveTTL
+	return nil
+}
+
+// isKMSv2ProviderHealthy checks if the KMSv2-Plugin is healthy.
+func isKMSv2ProviderHealthy(name string, response *envelopekmsv2.StatusResponse) error {
+	var errs []error
+	if response.Healthz != "ok" {
+		errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
+	}
+	if response.Version != envelopekmsv2.KMSAPIVersion {
+		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
+	}
+	if len(response.KeyID) == 0 {
+		errs = append(errs, fmt.Errorf("expected KMSv2 KeyID to be set, got %s", response.KeyID))
+	}
+
+	if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
+		return fmt.Errorf("kmsv2 Provider %s is not healthy, error: %w", name, err)
+	}
+	return nil
+}
+
+// loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
+func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
+	}
+	defer f.Close()
+
+	data, err := io.ReadAll(f)
+	if err != nil {
+		return nil, "", fmt.Errorf("could not read contents: %w", err)
+	}
+	if len(data) == 0 {
+		return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
+	}
+
+	scheme := runtime.NewScheme()
+	codecs := serializer.NewCodecFactory(scheme)
+	utilruntime.Must(apiserverconfig.AddToScheme(scheme))
+	utilruntime.Must(apiserverconfigv1.AddToScheme(scheme))
+
+	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, "", err
+	}
+	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	if !ok {
+		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+
+	return config, computeEncryptionConfigHash(data), validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
+}
+
+func prefixTransformersAndProbes(config apiserverconfig.ResourceConfiguration, stopCh <-chan struct{}) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
+	var transformers []value.PrefixTransformer
+	var probes []healthChecker
+	var kmsUsed kmsState
+
+	for _, provider := range config.Providers {
+		provider := provider
+		var (
+			transformer    value.PrefixTransformer
+			transformerErr error
+			probe          healthChecker
+			used           *kmsState
+		)
+
+		switch {
+		case provider.AESGCM != nil:
+			transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
+
+		case provider.AESCBC != nil:
+			transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
+
+		case provider.Secretbox != nil:
+			transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
+
+		case provider.KMS != nil:
+			transformer, probe, used, transformerErr = kmsPrefixTransformer(provider.KMS, stopCh)
+			if transformerErr == nil {
+				probes = append(probes, probe)
+				kmsUsed.v1Used = kmsUsed.v1Used || used.v1Used
+				kmsUsed.v2Used = kmsUsed.v2Used || used.v2Used
+
+				// calculate the maximum timeout for all KMS providers
+				kmsUsed.kmsTimeoutSum += used.kmsTimeoutSum
+			}
+
+		case provider.Identity != nil:
+			transformer = value.PrefixTransformer{
+				Transformer: identity.NewEncryptCheckTransformer(),
+				Prefix:      []byte{},
+			}
+
+		default:
+			return nil, nil, nil, errors.New("provider does not contain any of the expected providers: KMS, AESGCM, AESCBC, Secretbox, Identity")
+		}
+
+		if transformerErr != nil {
+			return nil, nil, nil, transformerErr
+		}
+
+		transformers = append(transformers, transformer)
+	}
+
+	return transformers, probes, &kmsUsed, nil
+}
+
+type blockTransformerFunc func(cipher.Block) value.Transformer
+
+func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("aes provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		key := key
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		keyData := keyData
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+		block, err := aes.NewCipher(key)
+		if err != nil {
+			return result, fmt.Errorf("error while creating cipher for named key %s: %s", keyData.Name, err)
+		}
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: fn(block),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(prefix),
+	}
+	return result, nil
+}
+
+func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("secretbox provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		key := key
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		keyData := keyData
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+
+		if len(key) != 32 {
+			return result, fmt.Errorf("expected key size 32 for secretbox provider, got %v", len(key))
+		}
+
+		keyArray := [32]byte{}
+		copy(keyArray[:], key)
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: secretbox.NewSecretboxTransformer(keyArray),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(secretboxTransformerPrefixV1),
+	}
+	return result, nil
+}
+
+var (
+	// The factory to create kms service. This is to make writing test easier.
+	envelopeServiceFactory = envelope.NewGRPCService
+
+	// The factory to create kmsv2 service.  Exported for integration tests.
+	EnvelopeKMSv2ServiceFactory = envelopekmsv2.NewGRPCService
+)
+
+type kmsState struct {
+	v1Used, v2Used bool
+	kmsTimeoutSum  time.Duration
+}
+
+func kmsPrefixTransformer(config *apiserverconfig.KMSConfiguration, stopCh <-chan struct{}) (value.PrefixTransformer, healthChecker, *kmsState, error) {
+	// we ignore the cancel func because this context should only be canceled when stopCh is closed
+	ctx, _ := wait.ContextForChannel(stopCh)
+
+	kmsName := config.Name
+	switch config.APIVersion {
+	case kmsAPIVersionV1:
+		envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
+		if err != nil {
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
+		}
+
+		probe := &kmsPluginProbe{
+			name:         kmsName,
+			ttl:          kmsPluginHealthzNegativeTTL,
+			service:      envelopeService,
+			l:            &sync.Mutex{},
+			lastResponse: &kmsPluginHealthzResponse{},
+		}
+
+		transformer := envelopePrefixTransformer(config, envelopeService, kmsTransformerPrefixV1)
+
+		return transformer, probe, &kmsState{
+			v1Used: true,
+			// for v1 we will do encrypt and decrypt for health check. Since these are serial operations, we will double the timeout.
+			kmsTimeoutSum: 2 * config.Timeout.Duration,
+		}, nil
+
+	case kmsAPIVersionV2:
+		if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
+		}
+
+		envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
+		if err != nil {
+			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
+		}
+
+		probe := &kmsv2PluginProbe{
+			name:         kmsName,
+			ttl:          kmsPluginHealthzNegativeTTL,
+			service:      envelopeService,
+			l:            &sync.Mutex{},
+			lastResponse: &kmsPluginHealthzResponse{},
+		}
+
+		// using AES-GCM by default for encrypting data with KMSv2
+		transformer := value.PrefixTransformer{
+			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewGCMTransformer),
+			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
+		}
+
+		return transformer, probe, &kmsState{
+			v2Used:        true,
+			kmsTimeoutSum: config.Timeout.Duration,
+		}, nil
+
+	default:
+		return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
+	}
+}
+
+func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
+	baseTransformerFunc := func(block cipher.Block) value.Transformer {
+		// v1.24: write using AES-CBC only but support reads via AES-CBC and AES-GCM (so we can move to AES-GCM)
+		// v1.25: write using AES-GCM only but support reads via AES-GCM and fallback to AES-CBC for backwards compatibility
+		// TODO(aramase): Post v1.25: We cannot drop CBC read support until we automate storage migration.
+		// We could have a release note that hard requires users to perform storage migration.
+		return unionTransformers{aestransformer.NewGCMTransformer(block), aestransformer.NewCBCTransformer(block)}
+	}
+
+	return value.PrefixTransformer{
+		Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
+		Prefix:      []byte(prefix + config.Name + ":"),
+	}
+}
+
+type unionTransformers []value.Transformer
+
+func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
+	var errs []error
+	for i := range u {
+		transformer := u[i]
+		result, stale, err := transformer.TransformFromStorage(ctx, data, dataCtx)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		// when i != 0, we have transformed the data from storage using the new transformer,
+		// we want to issue a write to etcd even if the contents of the data haven't changed
+		return result, stale || i != 0, nil
+	}
+	if err := utilerrors.Reduce(utilerrors.NewAggregate(errs)); err != nil {
+		return nil, false, err
+	}
+	return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
+}
+
+func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
+	return u[0].TransformToStorage(ctx, data, dataCtx)
+}
+
+// computeEncryptionConfigHash returns the expected hash for an encryption config file that has been loaded as bytes.
+// We use a hash instead of the raw file contents when tracking changes to avoid holding any encryption keys in memory outside of their associated transformers.
+// This hash must be used in-memory and not externalized to the process because it has no cross-release stability guarantees.
+func computeEncryptionConfigHash(data []byte) string {
+	return fmt.Sprintf("%x", sha256.Sum256(data))
+}
+
+var _ healthz.HealthChecker = &DynamicTransformers{}
+
+// DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
+// This struct must avoid locks (even read write locks) as it is inline to all calls to storage.
+type DynamicTransformers struct {
+	transformTracker *atomic.Value
+}
+
+type transformTracker struct {
+	transformerOverrides  map[schema.GroupResource]value.Transformer
+	kmsPluginHealthzCheck healthz.HealthChecker
+	closeTransformers     context.CancelFunc
+	kmsCloseGracePeriod   time.Duration
+}
+
+// NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
+func NewDynamicTransformers(
+	transformerOverrides map[schema.GroupResource]value.Transformer,
+	kmsPluginHealthzCheck healthz.HealthChecker,
+	closeTransformers context.CancelFunc,
+	kmsCloseGracePeriod time.Duration,
+) *DynamicTransformers {
+	dynamicTransformers := &DynamicTransformers{
+		transformTracker: &atomic.Value{},
+	}
+
+	tracker := &transformTracker{
+		transformerOverrides:  transformerOverrides,
+		kmsPluginHealthzCheck: kmsPluginHealthzCheck,
+		closeTransformers:     closeTransformers,
+		kmsCloseGracePeriod:   kmsCloseGracePeriod,
+	}
+	dynamicTransformers.transformTracker.Store(tracker)
+
+	return dynamicTransformers
+}
+
+// Check implements healthz.HealthChecker
+func (d *DynamicTransformers) Check(req *http.Request) error {
+	return d.transformTracker.Load().(*transformTracker).kmsPluginHealthzCheck.Check(req)
+}
+
+// Name implements healthz.HealthChecker
+func (d *DynamicTransformers) Name() string {
+	return kmsReloadHealthCheckName
+}
+
+// TransformerForResource returns the transformer for the given resource.
+func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
+	return &resourceTransformer{
+		resource:         resource,
+		transformTracker: d.transformTracker,
+	}
+}
+
+// Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
+func (d *DynamicTransformers) Set(
+	transformerOverrides map[schema.GroupResource]value.Transformer,
+	closeTransformers context.CancelFunc,
+	kmsPluginHealthzCheck healthz.HealthChecker,
+	kmsCloseGracePeriod time.Duration,
+) {
+	// store new values
+	newTransformTracker := &transformTracker{
+		transformerOverrides:  transformerOverrides,
+		closeTransformers:     closeTransformers,
+		kmsPluginHealthzCheck: kmsPluginHealthzCheck,
+		kmsCloseGracePeriod:   kmsCloseGracePeriod,
+	}
+
+	// update new transformer overrides
+	oldTransformTracker := d.transformTracker.Swap(newTransformTracker).(*transformTracker)
+
+	// close old transformers once we wait for grpc request to finish any in-flight requests.
+	// by the time we spawn this go routine, the new transformers have already been set and will be used for new requests.
+	// if the server starts shutting down during sleep duration then the transformers will correctly closed early because their lifetime is tied to the api-server drain notifier.
+	go func() {
+		time.Sleep(oldTransformTracker.kmsCloseGracePeriod)
+		oldTransformTracker.closeTransformers()
+	}()
+}
+
+var _ value.Transformer = &resourceTransformer{}
+
+type resourceTransformer struct {
+	resource         schema.GroupResource
+	transformTracker *atomic.Value
+}
+
+func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
+	return r.transformer().TransformFromStorage(ctx, data, dataCtx)
+}
+
+func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
+	return r.transformer().TransformToStorage(ctx, data, dataCtx)
+}
+
+func (r *resourceTransformer) transformer() value.Transformer {
+	transformer := r.transformTracker.Load().(*transformTracker).transformerOverrides[r.resource]
+	if transformer == nil {
+		return identity.NewEncryptCheckTransformer()
+	}
+	return transformer
+}
+
+type ResourceTransformers interface {
+	TransformerForResource(resource schema.GroupResource) value.Transformer
+}
+
+var _ ResourceTransformers = &DynamicTransformers{}
+var _ ResourceTransformers = &StaticTransformers{}
+
+type StaticTransformers map[schema.GroupResource]value.Transformer
+
+// StaticTransformers
+func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
+	transformer := s[resource]
+	if transformer == nil {
+		return identity.NewEncryptCheckTransformer()
+	}
+	return transformer
+}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -0,0 +1,265 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/fsnotify/fsnotify"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+)
+
+// workqueueKey is the dummy key used to process change in encryption config file.
+const workqueueKey = "key"
+
+// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file.
+type DynamicKMSEncryptionConfigContent struct {
+	name string
+
+	// filePath is the path of the file to read.
+	filePath string
+
+	// lastLoadedEncryptionConfigHash stores last successfully read encryption config file content.
+	lastLoadedEncryptionConfigHash string
+
+	// queue for processing changes in encryption config file.
+	queue workqueue.RateLimitingInterface
+
+	// dynamicTransformers updates the transformers when encryption config file changes.
+	dynamicTransformers *encryptionconfig.DynamicTransformers
+
+	// stopCh used here is a lifecycle signal of genericapiserver already drained while shutting down.
+	stopCh <-chan struct{}
+}
+
+// NewDynamicKMSEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
+func NewDynamicKMSEncryptionConfiguration(
+	name, filePath string,
+	dynamicTransformers *encryptionconfig.DynamicTransformers,
+	configContentHash string,
+	stopCh <-chan struct{},
+) *DynamicKMSEncryptionConfigContent {
+	encryptionConfig := &DynamicKMSEncryptionConfigContent{
+		name:                           name,
+		filePath:                       filePath,
+		lastLoadedEncryptionConfigHash: configContentHash,
+		dynamicTransformers:            dynamicTransformers,
+		stopCh:                         stopCh,
+		queue:                          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("%s-hot-reload", name)),
+	}
+	encryptionConfig.queue.Add(workqueueKey)
+
+	return encryptionConfig
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
+	defer utilruntime.HandleCrash()
+	defer d.queue.ShutDown()
+
+	klog.InfoS("Starting controller", "name", d.name)
+	defer klog.InfoS("Shutting down controller", "name", d.name)
+
+	// start worker for processing content
+	go wait.Until(d.runWorker, time.Second, ctx.Done())
+
+	// start the loop that watches the encryption config file until stopCh is closed.
+	go wait.Until(func() {
+		if err := d.watchEncryptionConfigFile(ctx.Done()); err != nil {
+			// if there is an error while setting up or handling the watches, this will ensure that we will process the config file.
+			defer d.queue.Add(workqueueKey)
+			klog.ErrorS(err, "Failed to watch encryption config file, will retry later")
+		}
+	}, time.Second, ctx.Done())
+
+	<-ctx.Done()
+}
+
+func (d *DynamicKMSEncryptionConfigContent) watchEncryptionConfigFile(stopCh <-chan struct{}) error {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		return fmt.Errorf("error creating fsnotify watcher: %w", err)
+	}
+	defer watcher.Close()
+
+	if err = watcher.Add(d.filePath); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
+	}
+
+	for {
+		select {
+		case event := <-watcher.Events:
+			if err := d.handleWatchEvent(event, watcher); err != nil {
+				return err
+			}
+		case err := <-watcher.Errors:
+			return fmt.Errorf("received fsnotify error: %w", err)
+		case <-stopCh:
+			return nil
+		}
+	}
+}
+
+func (d *DynamicKMSEncryptionConfigContent) handleWatchEvent(event fsnotify.Event, watcher *fsnotify.Watcher) error {
+	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
+	defer d.queue.Add(workqueueKey)
+
+	// return if file has not been removed or renamed.
+	if event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
+		return nil
+	}
+
+	if err := watcher.Remove(d.filePath); err != nil {
+		klog.V(2).InfoS("Failed to remove file watch, it may have been deleted", "file", d.filePath, "err", err)
+	}
+	if err := watcher.Add(d.filePath); err != nil {
+		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
+	}
+
+	return nil
+}
+
+// runWorker to process file content
+func (d *DynamicKMSEncryptionConfigContent) runWorker() {
+	for d.processNextWorkItem() {
+	}
+}
+
+// processNextWorkItem processes file content when there is a message in the queue.
+func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem() bool {
+	// key here is dummy item in the queue to trigger file content processing.
+	key, quit := d.queue.Get()
+	if quit {
+		return false
+	}
+	defer d.queue.Done(key)
+
+	var (
+		updatedEffectiveConfig  bool
+		err                     error
+		encryptionConfiguration *encryptionconfig.EncryptionConfiguration
+		configChanged           bool
+	)
+
+	// get context to close the new transformers.
+	ctx, closeTransformers := wait.ContextForChannel(d.stopCh)
+
+	defer func() {
+		// TODO: increment success metric when updatedEffectiveConfig=true
+
+		if !updatedEffectiveConfig {
+			// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
+			closeTransformers()
+		}
+		if err != nil {
+			// TODO: increment failure metric
+			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
+			// add dummy item back to the queue to trigger file content processing.
+			d.queue.AddRateLimited(key)
+		}
+	}()
+
+	encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
+	if err != nil {
+		return true
+	}
+	if !configChanged {
+		return true
+	}
+
+	if len(encryptionConfiguration.HealthChecks) != 1 {
+		err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
+		return true
+	}
+	// get healthz checks for all new KMS plugins.
+	if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
+		return true
+	}
+
+	// update transformers.
+	// when reload=true there must always be one healthz check.
+	d.dynamicTransformers.Set(
+		encryptionConfiguration.Transformers,
+		closeTransformers,
+		encryptionConfiguration.HealthChecks[0],
+		encryptionConfiguration.KMSCloseGracePeriod,
+	)
+
+	// update local copy of recent config content once update is successful.
+	d.lastLoadedEncryptionConfigHash = encryptionConfiguration.EncryptionFileContentHash
+	klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
+
+	updatedEffectiveConfig = true
+	return true
+}
+
+// loadEncryptionConfig processes the next set of content from the file.
+func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
+	encryptionConfiguration *encryptionconfig.EncryptionConfiguration,
+	configChanged bool,
+	err error,
+) {
+	// this code path will only execute if reload=true. So passing true explicitly.
+	encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(d.filePath, true, ctx.Done())
+	if err != nil {
+		return nil, false, err
+	}
+
+	// check if encryptionConfig is different from the current. Do nothing if they are the same.
+	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed", "name", d.name)
+		return nil, false, nil
+	}
+	return encryptionConfiguration, true, nil
+}
+
+func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
+	ctx context.Context,
+	kmsPluginHealthzCheck healthz.HealthChecker,
+	kmsPluginCloseGracePeriod time.Duration,
+) error {
+	// test if new transformers are healthy
+	var healthCheckError error
+
+	if kmsPluginCloseGracePeriod < 10*time.Second {
+		kmsPluginCloseGracePeriod = 10 * time.Second
+	}
+
+	pollErr := wait.PollImmediate(100*time.Millisecond, kmsPluginCloseGracePeriod, func() (bool, error) {
+		// create a fake http get request to health check endpoint
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("/healthz/%s", kmsPluginHealthzCheck.Name()), nil)
+		if err != nil {
+			return false, err
+		}
+
+		healthCheckError = kmsPluginHealthzCheck.Check(req)
+		return healthCheckError == nil, nil
+	})
+	if pollErr != nil {
+		return fmt.Errorf("health check for new transformers failed, polling error %v: %w", pollErr, healthCheckError)
+	}
+	klog.V(2).InfoS("Health check succeeded")
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -0,0 +1,476 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/registry/generic"
+	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	kmsconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
+	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
+	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	"k8s.io/klog/v2"
+)
+
+type EtcdOptions struct {
+	// The value of Paging on StorageConfig will be overridden by the
+	// calculated feature gate value.
+	StorageConfig                           storagebackend.Config
+	EncryptionProviderConfigFilepath        string
+	EncryptionProviderConfigAutomaticReload bool
+
+	EtcdServersOverrides []string
+
+	// To enable protobuf as storage format, it is enough
+	// to set it to "application/vnd.kubernetes.protobuf".
+	DefaultStorageMediaType string
+	DeleteCollectionWorkers int
+	EnableGarbageCollection bool
+
+	// Set EnableWatchCache to false to disable all watch caches
+	EnableWatchCache bool
+	// Set DefaultWatchCacheSize to zero to disable watch caches for those resources that have no explicit cache size set
+	DefaultWatchCacheSize int
+	// WatchCacheSizes represents override to a given resource
+	WatchCacheSizes []string
+
+	// complete guards fields that must be initialized via Complete before the Apply methods can be used.
+	complete               bool
+	resourceTransformers   encryptionconfig.ResourceTransformers
+	kmsPluginHealthzChecks []healthz.HealthChecker
+
+	// SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
+	// This allows multiple invocations of the Apply methods without duplication of said endpoints.
+	SkipHealthEndpoints bool
+}
+
+var storageTypes = sets.NewString(
+	storagebackend.StorageTypeETCD3,
+)
+
+func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
+	options := &EtcdOptions{
+		StorageConfig:           *backendConfig,
+		DefaultStorageMediaType: "application/json",
+		DeleteCollectionWorkers: 1,
+		EnableGarbageCollection: true,
+		EnableWatchCache:        true,
+		DefaultWatchCacheSize:   100,
+	}
+	options.StorageConfig.CountMetricPollPeriod = time.Minute
+	return options
+}
+
+func (s *EtcdOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	allErrors := []error{}
+	if len(s.StorageConfig.Transport.ServerList) == 0 {
+		allErrors = append(allErrors, fmt.Errorf("--etcd-servers must be specified"))
+	}
+
+	if s.StorageConfig.Type != storagebackend.StorageTypeUnset && !storageTypes.Has(s.StorageConfig.Type) {
+		allErrors = append(allErrors, fmt.Errorf("--storage-backend invalid, allowed values: %s. If not specified, it will default to 'etcd3'", strings.Join(storageTypes.List(), ", ")))
+	}
+
+	for _, override := range s.EtcdServersOverrides {
+		tokens := strings.Split(override, "#")
+		if len(tokens) != 2 {
+			allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
+			continue
+		}
+
+		apiresource := strings.Split(tokens[0], "/")
+		if len(apiresource) != 2 {
+			allErrors = append(allErrors, fmt.Errorf("--etcd-servers-overrides invalid, must be of format: group/resource#servers, where servers are URLs, semicolon separated"))
+			continue
+		}
+
+	}
+
+	if len(s.EncryptionProviderConfigFilepath) == 0 && s.EncryptionProviderConfigAutomaticReload {
+		allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
+	}
+
+	return allErrors
+}
+
+// AddFlags adds flags related to etcd storage for a specific APIServer to the specified FlagSet
+func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.StringSliceVar(&s.EtcdServersOverrides, "etcd-servers-overrides", s.EtcdServersOverrides, ""+
+		"Per-resource etcd servers overrides, comma separated. The individual override "+
+		"format: group/resource#servers, where servers are URLs, semicolon separated. "+
+		"Note that this applies only to resources compiled into this server binary. ")
+
+	fs.StringVar(&s.DefaultStorageMediaType, "storage-media-type", s.DefaultStorageMediaType, ""+
+		"The media type to use to store objects in storage. "+
+		"Some resources or storage backends may only support a specific media type and will ignore this setting. "+
+		"Supported media types: [application/json, application/yaml, application/vnd.kubernetes.protobuf]")
+	fs.IntVar(&s.DeleteCollectionWorkers, "delete-collection-workers", s.DeleteCollectionWorkers,
+		"Number of workers spawned for DeleteCollection call. These are used to speed up namespace cleanup.")
+
+	fs.BoolVar(&s.EnableGarbageCollection, "enable-garbage-collector", s.EnableGarbageCollection, ""+
+		"Enables the generic garbage collector. MUST be synced with the corresponding flag "+
+		"of the kube-controller-manager.")
+
+	fs.BoolVar(&s.EnableWatchCache, "watch-cache", s.EnableWatchCache,
+		"Enable watch caching in the apiserver")
+
+	fs.IntVar(&s.DefaultWatchCacheSize, "default-watch-cache-size", s.DefaultWatchCacheSize,
+		"Default watch cache size. If zero, watch cache will be disabled for resources that do not have a default watch size set.")
+
+	fs.MarkDeprecated("default-watch-cache-size",
+		"watch caches are sized automatically and this flag will be removed in a future version")
+
+	fs.StringSliceVar(&s.WatchCacheSizes, "watch-cache-sizes", s.WatchCacheSizes, ""+
+		"Watch cache size settings for some resources (pods, nodes, etc.), comma separated. "+
+		"The individual setting format: resource[.group]#size, where resource is lowercase plural (no version), "+
+		"group is omitted for resources of apiVersion v1 (the legacy core API) and included for others, "+
+		"and size is a number. This option is only meaningful for resources built into the apiserver, "+
+		"not ones defined by CRDs or aggregated from external servers, and is only consulted if the "+
+		"watch-cache is enabled. The only meaningful size setting to supply here is zero, which means to "+
+		"disable watch caching for the associated resource; all non-zero values are equivalent and mean "+
+		"to not disable watch caching for that resource")
+
+	fs.StringVar(&s.StorageConfig.Type, "storage-backend", s.StorageConfig.Type,
+		"The storage backend for persistence. Options: 'etcd3' (default).")
+
+	fs.StringSliceVar(&s.StorageConfig.Transport.ServerList, "etcd-servers", s.StorageConfig.Transport.ServerList,
+		"List of etcd servers to connect with (scheme://ip:port), comma separated.")
+
+	fs.StringVar(&s.StorageConfig.Prefix, "etcd-prefix", s.StorageConfig.Prefix,
+		"The prefix to prepend to all resource paths in etcd.")
+
+	fs.StringVar(&s.StorageConfig.Transport.KeyFile, "etcd-keyfile", s.StorageConfig.Transport.KeyFile,
+		"SSL key file used to secure etcd communication.")
+
+	fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile,
+		"SSL certification file used to secure etcd communication.")
+
+	fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile,
+		"SSL Certificate Authority file used to secure etcd communication.")
+
+	fs.StringVar(&s.EncryptionProviderConfigFilepath, "encryption-provider-config", s.EncryptionProviderConfigFilepath,
+		"The file containing configuration for encryption providers to be used for storing secrets in etcd")
+
+	fs.BoolVar(&s.EncryptionProviderConfigAutomaticReload, "encryption-provider-config-automatic-reload", s.EncryptionProviderConfigAutomaticReload,
+		"Determines if the file set by --encryption-provider-config should be automatically reloaded if the disk contents change. "+
+			"Setting this to true disables the ability to uniquely identify distinct KMS plugins via the API server healthz endpoints.")
+
+	fs.DurationVar(&s.StorageConfig.CompactionInterval, "etcd-compaction-interval", s.StorageConfig.CompactionInterval,
+		"The interval of compaction requests. If 0, the compaction request from apiserver is disabled.")
+
+	fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+
+		"Frequency of polling etcd for number of resources per type. 0 disables the metric collection.")
+
+	fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
+		"The interval of requests to poll etcd and update metric. 0 disables the metric collection")
+
+	fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
+		"The timeout to use when checking etcd health.")
+
+	fs.DurationVar(&s.StorageConfig.ReadycheckTimeout, "etcd-readycheck-timeout", s.StorageConfig.ReadycheckTimeout,
+		"The timeout to use when checking etcd readiness")
+
+	fs.Int64Var(&s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds, "lease-reuse-duration-seconds", s.StorageConfig.LeaseManagerConfig.ReuseDurationSeconds,
+		"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
+}
+
+// Complete must be called exactly once before using any of the Apply methods.  It is responsible for setting
+// up objects that must be created once and reused across multiple invocations such as storage transformers.
+// This method mutates the receiver (EtcdOptions).  It must never mutate the inputs.
+func (s *EtcdOptions) Complete(
+	storageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker,
+	stopCh <-chan struct{},
+	addPostStartHook func(name string, hook server.PostStartHookFunc) error,
+) error {
+	if s == nil {
+		return nil
+	}
+
+	if s.complete {
+		return fmt.Errorf("EtcdOptions.Complete called more than once")
+	}
+
+	if len(s.EncryptionProviderConfigFilepath) != 0 {
+		ctxTransformers, closeTransformers := wait.ContextForChannel(stopCh)
+		ctxServer, _ := wait.ContextForChannel(stopCh) // explicitly ignore cancel here because we do not own the server's lifecycle
+
+		encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, ctxTransformers.Done())
+		if err != nil {
+			// in case of error, we want to close partially initialized (if any) transformers
+			closeTransformers()
+			return err
+		}
+
+		// enable kms hot reload controller only if the config file is set to be automatically reloaded
+		if s.EncryptionProviderConfigAutomaticReload {
+			// with reload=true we will always have 1 health check
+			if len(encryptionConfiguration.HealthChecks) != 1 {
+				// in case of error, we want to close partially initialized (if any) transformers
+				closeTransformers()
+				return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
+			}
+
+			dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
+
+			s.resourceTransformers = dynamicTransformers
+			s.kmsPluginHealthzChecks = []healthz.HealthChecker{dynamicTransformers}
+
+			// add post start hook to start hot reload controller
+			// adding this hook here will ensure that it gets configured exactly once
+			err = addPostStartHook(
+				"start-encryption-provider-config-automatic-reload",
+				func(hookContext server.PostStartHookContext) error {
+					kmsConfigController := kmsconfigcontroller.NewDynamicKMSEncryptionConfiguration(
+						"kms-encryption-config",
+						s.EncryptionProviderConfigFilepath,
+						dynamicTransformers,
+						encryptionConfiguration.EncryptionFileContentHash,
+						ctxServer.Done(),
+					)
+
+					go kmsConfigController.Run(ctxServer)
+
+					return nil
+				},
+			)
+			if err != nil {
+				// in case of error, we want to close partially initialized (if any) transformers
+				closeTransformers()
+				return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
+			}
+		} else {
+			s.resourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
+			s.kmsPluginHealthzChecks = encryptionConfiguration.HealthChecks
+		}
+	}
+
+	s.StorageConfig.StorageObjectCountTracker = storageObjectCountTracker
+
+	s.complete = true
+
+	return nil
+}
+
+// ApplyTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
+func (s *EtcdOptions) ApplyTo(c *server.Config) error {
+	if s == nil {
+		return nil
+	}
+
+	return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: s.StorageConfig}, c)
+}
+
+// ApplyWithStorageFactoryTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
+func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error {
+	if s == nil {
+		return nil
+	}
+
+	if !s.complete {
+		return fmt.Errorf("EtcdOptions.Apply called without completion")
+	}
+
+	if !s.SkipHealthEndpoints {
+		if err := s.addEtcdHealthEndpoint(c); err != nil {
+			return err
+		}
+	}
+
+	if s.resourceTransformers != nil {
+		factory = &transformerStorageFactory{
+			delegate:             factory,
+			resourceTransformers: s.resourceTransformers,
+		}
+	}
+
+	c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
+	return nil
+}
+
+func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
+	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
+	if err != nil {
+		return err
+	}
+	c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
+		return healthCheck()
+	}))
+
+	readyCheck, err := storagefactory.CreateReadyCheck(s.StorageConfig, c.DrainedNotify())
+	if err != nil {
+		return err
+	}
+	c.AddReadyzChecks(healthz.NamedCheck("etcd-readiness", func(r *http.Request) error {
+		return readyCheck()
+	}))
+
+	c.AddHealthChecks(s.kmsPluginHealthzChecks...)
+
+	return nil
+}
+
+type StorageFactoryRestOptionsFactory struct {
+	Options        EtcdOptions
+	StorageFactory serverstorage.StorageFactory
+}
+
+func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
+	storageConfig, err := f.StorageFactory.NewConfig(resource)
+	if err != nil {
+		return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
+	}
+
+	ret := generic.RESTOptions{
+		StorageConfig:             storageConfig,
+		Decorator:                 generic.UndecoratedStorage,
+		DeleteCollectionWorkers:   f.Options.DeleteCollectionWorkers,
+		EnableGarbageCollection:   f.Options.EnableGarbageCollection,
+		ResourcePrefix:            f.StorageFactory.ResourcePrefix(resource),
+		CountMetricPollPeriod:     f.Options.StorageConfig.CountMetricPollPeriod,
+		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
+	}
+
+	if f.Options.EnableWatchCache {
+		sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
+		if err != nil {
+			return generic.RESTOptions{}, err
+		}
+		size, ok := sizes[resource]
+		if ok && size > 0 {
+			klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
+		}
+		if ok && size <= 0 {
+			klog.V(3).InfoS("Not using watch cache", "resource", resource)
+			ret.Decorator = generic.UndecoratedStorage
+		} else {
+			klog.V(3).InfoS("Using watch cache", "resource", resource)
+			ret.Decorator = genericregistry.StorageWithCacher()
+		}
+	}
+
+	return ret, nil
+}
+
+// ParseWatchCacheSizes turns a list of cache size values into a map of group resources
+// to requested sizes.
+func ParseWatchCacheSizes(cacheSizes []string) (map[schema.GroupResource]int, error) {
+	watchCacheSizes := make(map[schema.GroupResource]int)
+	for _, c := range cacheSizes {
+		tokens := strings.Split(c, "#")
+		if len(tokens) != 2 {
+			return nil, fmt.Errorf("invalid value of watch cache size: %s", c)
+		}
+
+		size, err := strconv.Atoi(tokens[1])
+		if err != nil {
+			return nil, fmt.Errorf("invalid size of watch cache size: %s", c)
+		}
+		if size < 0 {
+			return nil, fmt.Errorf("watch cache size cannot be negative: %s", c)
+		}
+		watchCacheSizes[schema.ParseGroupResource(tokens[0])] = size
+	}
+	return watchCacheSizes, nil
+}
+
+// WriteWatchCacheSizes turns a map of cache size values into a list of string specifications.
+func WriteWatchCacheSizes(watchCacheSizes map[schema.GroupResource]int) ([]string, error) {
+	var cacheSizes []string
+
+	for resource, size := range watchCacheSizes {
+		if size < 0 {
+			return nil, fmt.Errorf("watch cache size cannot be negative for resource %s", resource)
+		}
+		cacheSizes = append(cacheSizes, fmt.Sprintf("%s#%d", resource.String(), size))
+	}
+	return cacheSizes, nil
+}
+
+var _ serverstorage.StorageFactory = &SimpleStorageFactory{}
+
+// SimpleStorageFactory provides a StorageFactory implementation that should be used when different
+// resources essentially share the same storage config (as defined by the given storagebackend.Config).
+// It assumes the resources are stored at a path that is purely based on the schema.GroupResource.
+// Users that need flexibility and per resource overrides should use DefaultStorageFactory instead.
+type SimpleStorageFactory struct {
+	StorageConfig storagebackend.Config
+}
+
+func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+	return s.StorageConfig.ForResource(resource), nil
+}
+
+func (s *SimpleStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
+	return resource.Group + "/" + resource.Resource
+}
+
+func (s *SimpleStorageFactory) Backends() []serverstorage.Backend {
+	// nothing should ever call this method but we still provide a functional implementation
+	return serverstorage.Backends(s.StorageConfig)
+}
+
+var _ serverstorage.StorageFactory = &transformerStorageFactory{}
+
+type transformerStorageFactory struct {
+	delegate             serverstorage.StorageFactory
+	resourceTransformers encryptionconfig.ResourceTransformers
+}
+
+func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+	config, err := t.delegate.NewConfig(resource)
+	if err != nil {
+		return nil, err
+	}
+
+	configCopy := *config
+	resourceConfig := configCopy.Config
+	resourceConfig.Transformer = t.resourceTransformers.TransformerForResource(resource)
+	configCopy.Config = resourceConfig
+
+	return &configCopy, nil
+}
+
+func (t *transformerStorageFactory) ResourcePrefix(resource schema.GroupResource) string {
+	return t.delegate.ResourcePrefix(resource)
+}
+
+func (t *transformerStorageFactory) Backends() []serverstorage.Backend {
+	return t.delegate.Backends()
+}

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/server"
+)
+
+type FeatureOptions struct {
+	EnableProfiling           bool
+	EnableContentionProfiling bool
+}
+
+func NewFeatureOptions() *FeatureOptions {
+	defaults := server.NewConfig(serializer.CodecFactory{})
+
+	return &FeatureOptions{
+		EnableProfiling:           defaults.EnableProfiling,
+		EnableContentionProfiling: defaults.EnableContentionProfiling,
+	}
+}
+
+func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.BoolVar(&o.EnableProfiling, "profiling", o.EnableProfiling,
+		"Enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&o.EnableContentionProfiling, "contention-profiling", o.EnableContentionProfiling,
+		"Enable lock contention profiling, if profiling is enabled")
+}
+
+func (o *FeatureOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	c.EnableProfiling = o.EnableProfiling
+	c.EnableContentionProfiling = o.EnableContentionProfiling
+
+	return nil
+}
+
+func (o *FeatureOptions) Validate() []error {
+	if o == nil {
+		return nil
+	}
+
+	errs := []error{}
+	return errs
+}

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -0,0 +1,172 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/spf13/pflag"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/apiserver/pkg/util/feature"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/component-base/featuregate"
+	"k8s.io/klog/v2"
+)
+
+// RecommendedOptions contains the recommended options for running an API server.
+// If you add something to this list, it should be in a logical grouping.
+// Each of them can be nil to leave the feature unconfigured on ApplyTo.
+type RecommendedOptions struct {
+	Etcd           *EtcdOptions
+	SecureServing  *SecureServingOptionsWithLoopback
+	Authentication *DelegatingAuthenticationOptions
+	Authorization  *DelegatingAuthorizationOptions
+	Audit          *AuditOptions
+	Features       *FeatureOptions
+	CoreAPI        *CoreAPIOptions
+
+	// FeatureGate is a way to plumb feature gate through if you have them.
+	FeatureGate featuregate.FeatureGate
+	// ExtraAdmissionInitializers is called once after all ApplyTo from the options above, to pass the returned
+	// admission plugin initializers to Admission.ApplyTo.
+	ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
+	Admission                  *AdmissionOptions
+	// API Server Egress Selector is used to control outbound traffic from the API Server
+	EgressSelector *EgressSelectorOptions
+	// Traces contains options to control distributed request tracing.
+	Traces *TracingOptions
+}
+
+func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
+	sso := NewSecureServingOptions()
+
+	// We are composing recommended options for an aggregated api-server,
+	// whose client is typically a proxy multiplexing many operations ---
+	// notably including long-running ones --- into one HTTP/2 connection
+	// into this server.  So allow many concurrent operations.
+	sso.HTTP2MaxStreamsPerConnection = 1000
+
+	return &RecommendedOptions{
+		Etcd:           NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)),
+		SecureServing:  sso.WithLoopback(),
+		Authentication: NewDelegatingAuthenticationOptions(),
+		Authorization:  NewDelegatingAuthorizationOptions(),
+		Audit:          NewAuditOptions(),
+		Features:       NewFeatureOptions(),
+		CoreAPI:        NewCoreAPIOptions(),
+		// Wired a global by default that sadly people will abuse to have different meanings in different repos.
+		// Please consider creating your own FeatureGate so you can have a consistent meaning for what a variable contains
+		// across different repos.  Future you will thank you.
+		FeatureGate:                feature.DefaultFeatureGate,
+		ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
+		Admission:                  NewAdmissionOptions(),
+		EgressSelector:             NewEgressSelectorOptions(),
+		Traces:                     NewTracingOptions(),
+	}
+}
+
+func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
+	o.Etcd.AddFlags(fs)
+	o.SecureServing.AddFlags(fs)
+	o.Authentication.AddFlags(fs)
+	o.Authorization.AddFlags(fs)
+	o.Audit.AddFlags(fs)
+	o.Features.AddFlags(fs)
+	o.CoreAPI.AddFlags(fs)
+	o.Admission.AddFlags(fs)
+	o.EgressSelector.AddFlags(fs)
+	o.Traces.AddFlags(fs)
+}
+
+// ApplyTo adds RecommendedOptions to the server configuration.
+// pluginInitializers can be empty, it is only need for additional initializers.
+func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
+	if err := o.Etcd.Complete(config.Config.StorageObjectCountTracker, config.Config.DrainedNotify(), config.Config.AddPostStartHook); err != nil {
+		return err
+	}
+	if err := o.Etcd.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
+		return err
+	}
+	if err := o.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
+		return err
+	}
+	if err := o.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
+		return err
+	}
+	if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
+		return err
+	}
+	if err := o.Audit.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.Features.ApplyTo(&config.Config); err != nil {
+		return err
+	}
+	if err := o.CoreAPI.ApplyTo(config); err != nil {
+		return err
+	}
+	if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
+		return err
+	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
+		return err
+	}
+	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
+		if config.ClientConfig != nil {
+			if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
+				return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight)
+
+			}
+			config.FlowControl = utilflowcontrol.New(
+				config.SharedInformerFactory,
+				kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(),
+				config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
+				config.RequestTimeout/4,
+			)
+		} else {
+			klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled")
+		}
+	}
+	return nil
+}
+
+func (o *RecommendedOptions) Validate() []error {
+	errors := []error{}
+	errors = append(errors, o.Etcd.Validate()...)
+	errors = append(errors, o.SecureServing.Validate()...)
+	errors = append(errors, o.Authentication.Validate()...)
+	errors = append(errors, o.Authorization.Validate()...)
+	errors = append(errors, o.Audit.Validate()...)
+	errors = append(errors, o.Features.Validate()...)
+	errors = append(errors, o.CoreAPI.Validate()...)
+	errors = append(errors, o.Admission.Validate()...)
+	errors = append(errors, o.EgressSelector.Validate()...)
+	errors = append(errors, o.Traces.Validate()...)
+
+	return errors
+}

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -0,0 +1,261 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/server"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+
+	"github.com/spf13/pflag"
+)
+
+// ServerRunOptions contains the options while running a generic api server.
+type ServerRunOptions struct {
+	AdvertiseAddress net.IP
+
+	CorsAllowedOriginList       []string
+	HSTSDirectives              []string
+	ExternalHost                string
+	MaxRequestsInFlight         int
+	MaxMutatingRequestsInFlight int
+	RequestTimeout              time.Duration
+	GoawayChance                float64
+	LivezGracePeriod            time.Duration
+	MinRequestTimeout           int
+	ShutdownDelayDuration       time.Duration
+	// We intentionally did not add a flag for this option. Users of the
+	// apiserver library can wire it to a flag.
+	JSONPatchMaxCopyBytes int64
+	// The limit on the request body size that would be accepted and
+	// decoded in a write request. 0 means no limit.
+	// We intentionally did not add a flag for this option. Users of the
+	// apiserver library can wire it to a flag.
+	MaxRequestBodyBytes       int64
+	EnablePriorityAndFairness bool
+
+	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
+	// Server during the graceful termination of the apiserver. If true, we wait
+	// for non longrunning requests in flight to be drained and then initiate a
+	// shutdown of the HTTP Server. If false, we initiate a shutdown of the HTTP
+	// Server as soon as ShutdownDelayDuration has elapsed.
+	// If enabled, after ShutdownDelayDuration elapses, any incoming request is
+	// rejected with a 429 status code and a 'Retry-After' response.
+	ShutdownSendRetryAfter bool
+}
+
+func NewServerRunOptions() *ServerRunOptions {
+	defaults := server.NewConfig(serializer.CodecFactory{})
+	return &ServerRunOptions{
+		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
+		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
+		RequestTimeout:              defaults.RequestTimeout,
+		LivezGracePeriod:            defaults.LivezGracePeriod,
+		MinRequestTimeout:           defaults.MinRequestTimeout,
+		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
+		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
+		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
+		EnablePriorityAndFairness:   true,
+		ShutdownSendRetryAfter:      false,
+	}
+}
+
+// ApplyTo applies the run options to the method receiver and returns self
+func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
+	c.CorsAllowedOriginList = s.CorsAllowedOriginList
+	c.HSTSDirectives = s.HSTSDirectives
+	c.ExternalAddress = s.ExternalHost
+	c.MaxRequestsInFlight = s.MaxRequestsInFlight
+	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
+	c.LivezGracePeriod = s.LivezGracePeriod
+	c.RequestTimeout = s.RequestTimeout
+	c.GoawayChance = s.GoawayChance
+	c.MinRequestTimeout = s.MinRequestTimeout
+	c.ShutdownDelayDuration = s.ShutdownDelayDuration
+	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
+	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
+	c.PublicAddress = s.AdvertiseAddress
+	c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
+
+	return nil
+}
+
+// DefaultAdvertiseAddress sets the field AdvertiseAddress if unset. The field will be set based on the SecureServingOptions.
+func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions) error {
+	if secure == nil {
+		return nil
+	}
+
+	if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
+		hostIP, err := secure.DefaultExternalAddress()
+		if err != nil {
+			return fmt.Errorf("Unable to find suitable network address.error='%v'. "+
+				"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
+		}
+		s.AdvertiseAddress = hostIP
+	}
+
+	return nil
+}
+
+// Validate checks validation of ServerRunOptions
+func (s *ServerRunOptions) Validate() []error {
+	errors := []error{}
+
+	if s.LivezGracePeriod < 0 {
+		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
+	}
+
+	if s.MaxRequestsInFlight < 0 {
+		errors = append(errors, fmt.Errorf("--max-requests-inflight can not be negative value"))
+	}
+	if s.MaxMutatingRequestsInFlight < 0 {
+		errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight can not be negative value"))
+	}
+
+	if s.RequestTimeout.Nanoseconds() < 0 {
+		errors = append(errors, fmt.Errorf("--request-timeout can not be negative value"))
+	}
+
+	if s.GoawayChance < 0 || s.GoawayChance > 0.02 {
+		errors = append(errors, fmt.Errorf("--goaway-chance can not be less than 0 or greater than 0.02"))
+	}
+
+	if s.MinRequestTimeout < 0 {
+		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
+	}
+
+	if s.ShutdownDelayDuration < 0 {
+		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
+	}
+
+	if s.JSONPatchMaxCopyBytes < 0 {
+		errors = append(errors, fmt.Errorf("ServerRunOptions.JSONPatchMaxCopyBytes can not be negative value"))
+	}
+
+	if s.MaxRequestBodyBytes < 0 {
+		errors = append(errors, fmt.Errorf("ServerRunOptions.MaxRequestBodyBytes can not be negative value"))
+	}
+
+	if err := validateHSTSDirectives(s.HSTSDirectives); err != nil {
+		errors = append(errors, err)
+	}
+	return errors
+}
+
+func validateHSTSDirectives(hstsDirectives []string) error {
+	// HSTS Headers format: Strict-Transport-Security:max-age=expireTime [;includeSubDomains] [;preload]
+	// See https://tools.ietf.org/html/rfc6797#section-6.1 for more information
+	allErrors := []error{}
+	for _, hstsDirective := range hstsDirectives {
+		if len(strings.TrimSpace(hstsDirective)) == 0 {
+			allErrors = append(allErrors, fmt.Errorf("empty value in strict-transport-security-directives"))
+			continue
+		}
+		if hstsDirective != "includeSubDomains" && hstsDirective != "preload" {
+			maxAgeDirective := strings.Split(hstsDirective, "=")
+			if len(maxAgeDirective) != 2 || maxAgeDirective[0] != "max-age" {
+				allErrors = append(allErrors, fmt.Errorf("--strict-transport-security-directives invalid, allowed values: max-age=expireTime, includeSubDomains, preload. see https://tools.ietf.org/html/rfc6797#section-6.1 for more information"))
+			}
+		}
+	}
+	return errors.NewAggregate(allErrors)
+}
+
+// AddUniversalFlags adds flags for a specific APIServer to the specified FlagSet
+func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
+	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
+	// arrange these text blocks sensibly. Grrr.
+
+	fs.IPVar(&s.AdvertiseAddress, "advertise-address", s.AdvertiseAddress, ""+
+		"The IP address on which to advertise the apiserver to members of the cluster. This "+
+		"address must be reachable by the rest of the cluster. If blank, the --bind-address "+
+		"will be used. If --bind-address is unspecified, the host's default interface will "+
+		"be used.")
+
+	fs.StringSliceVar(&s.CorsAllowedOriginList, "cors-allowed-origins", s.CorsAllowedOriginList, ""+
+		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
+		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")
+
+	fs.StringSliceVar(&s.HSTSDirectives, "strict-transport-security-directives", s.HSTSDirectives, ""+
+		"List of directives for HSTS, comma separated. If this list is empty, then HSTS directives will not "+
+		"be added. Example: 'max-age=31536000,includeSubDomains,preload'")
+
+	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
+		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
+
+	deprecatedMasterServiceNamespace := metav1.NamespaceDefault
+	fs.StringVar(&deprecatedMasterServiceNamespace, "master-service-namespace", deprecatedMasterServiceNamespace, ""+
+		"DEPRECATED: the namespace from which the Kubernetes master services should be injected into pods.")
+	fs.MarkDeprecated("master-service-namespace", "This flag will be removed in v1.27")
+
+	fs.IntVar(&s.MaxRequestsInFlight, "max-requests-inflight", s.MaxRequestsInFlight, ""+
+		"This and --max-mutating-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of non-mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
+
+	fs.IntVar(&s.MaxMutatingRequestsInFlight, "max-mutating-requests-inflight", s.MaxMutatingRequestsInFlight, ""+
+		"This and --max-requests-inflight are summed to determine the server's total concurrency limit "+
+		"(which must be positive) if --enable-priority-and-fairness is true. "+
+		"Otherwise, this flag limits the maximum number of mutating requests in flight, "+
+		"or a zero value disables the limit completely.")
+
+	fs.DurationVar(&s.RequestTimeout, "request-timeout", s.RequestTimeout, ""+
+		"An optional field indicating the duration a handler must keep a request open before timing "+
+		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
+		"--min-request-timeout for specific types of requests.")
+
+	fs.Float64Var(&s.GoawayChance, "goaway-chance", s.GoawayChance, ""+
+		"To prevent HTTP/2 clients from getting stuck on a single apiserver, randomly close a connection (GOAWAY). "+
+		"The client's other in-flight requests won't be affected, and the client will reconnect, likely landing on a different apiserver after going through the load balancer again. "+
+		"This argument sets the fraction of requests that will be sent a GOAWAY. Clusters with single apiservers, or which don't use a load balancer, should NOT enable this. "+
+		"Min is 0 (off), Max is .02 (1/50 requests); .001 (1/1000) is a recommended starting point.")
+
+	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
+		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
+		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
+		"that unfinished post-start hooks will complete successfully and therefore return true.")
+
+	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
+		"An optional field indicating the minimum number of seconds a handler must keep "+
+		"a request open before timing it out. Currently only honored by the watch request "+
+		"handler, which picks a randomized value above this number as the connection timeout, "+
+		"to spread out load.")
+
+	fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
+		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
+
+	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
+		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
+		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
+		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
+
+	fs.BoolVar(&s.ShutdownSendRetryAfter, "shutdown-send-retry-after", s.ShutdownSendRetryAfter, ""+
+		"If true the HTTP Server will continue listening until all non long running request(s) in flight have been drained, "+
+		"during this window all incoming requests will be rejected with a status code 429 and a 'Retry-After' response header, "+
+		"in addition 'Connection: close' response header is set in order to tear down the TCP connection when idle.")
+
+	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
+}

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -0,0 +1,384 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"path"
+	"strconv"
+	"strings"
+	"syscall"
+
+	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+	netutils "k8s.io/utils/net"
+
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	certutil "k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/keyutil"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+type SecureServingOptions struct {
+	BindAddress net.IP
+	// BindPort is ignored when Listener is set, will serve https even with 0.
+	BindPort int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+	// Required set to true means that BindPort cannot be zero.
+	Required bool
+	// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this
+	// is set to BindAddress if the later no loopback, or to the first host interface address.
+	ExternalAddress net.IP
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
+
+	// ServerCert is the TLS cert info for serving secure traffic
+	ServerCert GeneratableKeyCert
+	// SNICertKeys are named CertKeys for serving secure traffic with SNI support.
+	SNICertKeys []cliflag.NamedCertKey
+	// CipherSuites is the list of allowed cipher suites for the server.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	CipherSuites []string
+	// MinTLSVersion is the minimum TLS version supported.
+	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
+	MinTLSVersion string
+
+	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
+	// A value of zero means to use the default provided by golang's HTTP/2 support.
+	HTTP2MaxStreamsPerConnection int
+
+	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
+	// more than one instance to bind on the same address and port.
+	PermitPortSharing bool
+
+	// PermitAddressSharing controls if SO_REUSEADDR is used when binding the port.
+	PermitAddressSharing bool
+}
+
+type CertKey struct {
+	// CertFile is a file containing a PEM-encoded certificate, and possibly the complete certificate chain
+	CertFile string
+	// KeyFile is a file containing a PEM-encoded private key for the certificate specified by CertFile
+	KeyFile string
+}
+
+type GeneratableKeyCert struct {
+	// CertKey allows setting an explicit cert/key file to use.
+	CertKey CertKey
+
+	// CertDirectory specifies a directory to write generated certificates to if CertFile/KeyFile aren't explicitly set.
+	// PairName is used to determine the filenames within CertDirectory.
+	// If CertDirectory and PairName are not set, an in-memory certificate will be generated.
+	CertDirectory string
+	// PairName is the name which will be used with CertDirectory to make a cert and key filenames.
+	// It becomes CertDirectory/PairName.crt and CertDirectory/PairName.key
+	PairName string
+
+	// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set.
+	GeneratedCert dynamiccertificates.CertKeyContentProvider
+
+	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
+	// The format is:
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.crt
+	// <host>_<ip>-<ip>_<alternateDNS>-<alternateDNS>.key
+	FixtureDirectory string
+}
+
+func NewSecureServingOptions() *SecureServingOptions {
+	return &SecureServingOptions{
+		BindAddress: netutils.ParseIPSloppy("0.0.0.0"),
+		BindPort:    443,
+		ServerCert: GeneratableKeyCert{
+			PairName:      "apiserver",
+			CertDirectory: "apiserver.local.config/certificates",
+		},
+	}
+}
+
+func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) {
+	if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() {
+		return s.ExternalAddress, nil
+	}
+	return utilnet.ResolveBindAddress(s.BindAddress)
+}
+
+func (s *SecureServingOptions) Validate() []error {
+	if s == nil {
+		return nil
+	}
+
+	errors := []error{}
+
+	if s.Required && s.BindPort < 1 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("--secure-port %v must be between 1 and 65535, inclusive. It cannot be turned off with 0", s.BindPort))
+	} else if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort))
+	}
+
+	if (len(s.ServerCert.CertKey.CertFile) != 0 || len(s.ServerCert.CertKey.KeyFile) != 0) && s.ServerCert.GeneratedCert != nil {
+		errors = append(errors, fmt.Errorf("cert/key file and in-memory certificate cannot both be set"))
+	}
+
+	return errors
+}
+
+func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
+		"The IP address on which to listen for the --secure-port port. The "+
+		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
+		"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces will be used.")
+
+	desc := "The port on which to serve HTTPS with authentication and authorization."
+	if s.Required {
+		desc += " It cannot be switched off with 0."
+	} else {
+		desc += " If 0, don't serve HTTPS at all."
+	}
+	fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc)
+
+	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
+		"The directory where the TLS certs are located. "+
+		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
+
+	fs.StringVar(&s.ServerCert.CertKey.CertFile, "tls-cert-file", s.ServerCert.CertKey.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert). If HTTPS serving is enabled, and --tls-cert-file and "+
+		"--tls-private-key-file are not provided, a self-signed certificate and key "+
+		"are generated for the public address and saved to the directory specified by --cert-dir.")
+
+	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
+		"File containing the default x509 private key matching --tls-cert-file.")
+
+	tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
+	tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
+	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
+		"Comma-separated list of cipher suites for the server. "+
+			"If omitted, the default Go cipher suites will be used. \n"+
+			"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
+			"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
+
+	tlsPossibleVersions := cliflag.TLSPossibleVersions()
+	fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
+		"Minimum TLS version supported. "+
+			"Possible values: "+strings.Join(tlsPossibleVersions, ", "))
+
+	fs.Var(cliflag.NewNamedCertKeyArray(&s.SNICertKeys), "tls-sni-cert-key", ""+
+		"A pair of x509 certificate and private key file paths, optionally suffixed with a list of "+
+		"domain patterns which are fully qualified domain names, possibly with prefixed wildcard "+
+		"segments. The domain patterns also allow IP addresses, but IPs should only be used if "+
+		"the apiserver has visibility to the IP address requested by a client. "+
+		"If no domain patterns are provided, the names of the certificate are "+
+		"extracted. Non-wildcard matches trump over wildcard matches, explicit domain patterns "+
+		"trump over extracted names. For multiple key/certificate pairs, use the "+
+		"--tls-sni-cert-key multiple times. "+
+		"Examples: \"example.crt,example.key\" or \"foo.crt,foo.key:*.foo.com,foo.com\".")
+
+	fs.IntVar(&s.HTTP2MaxStreamsPerConnection, "http2-max-streams-per-connection", s.HTTP2MaxStreamsPerConnection, ""+
+		"The limit that the server gives to clients for "+
+		"the maximum number of streams in an HTTP/2 connection. "+
+		"Zero means to use golang's default.")
+
+	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
+		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
+			"more than one instance to bind on the same address and port. [default=false]")
+
+	fs.BoolVar(&s.PermitAddressSharing, "permit-address-sharing", s.PermitAddressSharing,
+		"If true, SO_REUSEADDR will be used when binding the port. This allows binding "+
+			"to wildcard IPs like 0.0.0.0 and specific IPs in parallel, and it avoids waiting "+
+			"for the kernel to release sockets in TIME_WAIT state. [default=false]")
+}
+
+// ApplyTo fills up serving information in the server configuration.
+func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error {
+	if s == nil {
+		return nil
+	}
+	if s.BindPort <= 0 && s.Listener == nil {
+		return nil
+	}
+
+	if s.Listener == nil {
+		var err error
+		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
+
+		c := net.ListenConfig{}
+
+		ctls := multipleControls{}
+		if s.PermitPortSharing {
+			ctls = append(ctls, permitPortReuse)
+		}
+		if s.PermitAddressSharing {
+			ctls = append(ctls, permitAddressReuse)
+		}
+		if len(ctls) > 0 {
+			c.Control = ctls.Control
+		}
+
+		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	} else {
+		if _, ok := s.Listener.Addr().(*net.TCPAddr); !ok {
+			return fmt.Errorf("failed to parse ip and port from listener")
+		}
+		s.BindPort = s.Listener.Addr().(*net.TCPAddr).Port
+		s.BindAddress = s.Listener.Addr().(*net.TCPAddr).IP
+	}
+
+	*config = &server.SecureServingInfo{
+		Listener:                     s.Listener,
+		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
+	}
+	c := *config
+
+	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
+	// load main cert
+	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
+		var err error
+		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
+		if err != nil {
+			return err
+		}
+	} else if s.ServerCert.GeneratedCert != nil {
+		c.Cert = s.ServerCert.GeneratedCert
+	}
+
+	if len(s.CipherSuites) != 0 {
+		cipherSuites, err := cliflag.TLSCipherSuites(s.CipherSuites)
+		if err != nil {
+			return err
+		}
+		c.CipherSuites = cipherSuites
+	}
+
+	var err error
+	c.MinTLSVersion, err = cliflag.TLSVersion(s.MinTLSVersion)
+	if err != nil {
+		return err
+	}
+
+	// load SNI certs
+	namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys))
+	for _, nck := range s.SNICertKeys {
+		tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...)
+		namedTLSCerts = append(namedTLSCerts, tlsCert)
+		if err != nil {
+			return fmt.Errorf("failed to load SNI cert and key: %v", err)
+		}
+	}
+	c.SNICerts = namedTLSCerts
+
+	return nil
+}
+
+func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress string, alternateDNS []string, alternateIPs []net.IP) error {
+	if s == nil || (s.BindPort == 0 && s.Listener == nil) {
+		return nil
+	}
+	keyCert := &s.ServerCert.CertKey
+	if len(keyCert.CertFile) != 0 || len(keyCert.KeyFile) != 0 {
+		return nil
+	}
+
+	canReadCertAndKey := false
+	if len(s.ServerCert.CertDirectory) > 0 {
+		if len(s.ServerCert.PairName) == 0 {
+			return fmt.Errorf("PairName is required if CertDirectory is set")
+		}
+		keyCert.CertFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".crt")
+		keyCert.KeyFile = path.Join(s.ServerCert.CertDirectory, s.ServerCert.PairName+".key")
+		if canRead, err := certutil.CanReadCertAndKey(keyCert.CertFile, keyCert.KeyFile); err != nil {
+			return err
+		} else {
+			canReadCertAndKey = canRead
+		}
+	}
+
+	if !canReadCertAndKey {
+		// add either the bind address or localhost to the valid alternates
+		if s.BindAddress.IsUnspecified() {
+			alternateDNS = append(alternateDNS, "localhost")
+		} else {
+			alternateIPs = append(alternateIPs, s.BindAddress)
+		}
+
+		if cert, key, err := certutil.GenerateSelfSignedCertKeyWithFixtures(publicAddress, alternateIPs, alternateDNS, s.ServerCert.FixtureDirectory); err != nil {
+			return fmt.Errorf("unable to generate self signed cert: %v", err)
+		} else if len(keyCert.CertFile) > 0 && len(keyCert.KeyFile) > 0 {
+			if err := certutil.WriteCert(keyCert.CertFile, cert); err != nil {
+				return err
+			}
+			if err := keyutil.WriteKey(keyCert.KeyFile, key); err != nil {
+				return err
+			}
+			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
+		} else {
+			s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Generated self-signed cert in-memory")
+		}
+	}
+
+	return nil
+}
+
+func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
+	if len(network) == 0 {
+		network = "tcp"
+	}
+
+	ln, err := config.Listen(context.TODO(), network, addr)
+	if err != nil {
+		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
+	}
+
+	// get port
+	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
+	if !ok {
+		ln.Close()
+		return nil, 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
+	}
+
+	return ln, tcpAddr.Port, nil
+}
+
+type multipleControls []func(network, addr string, conn syscall.RawConn) error
+
+func (mcs multipleControls) Control(network, addr string, conn syscall.RawConn) error {
+	for _, c := range mcs {
+		if err := c(network, addr, conn); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -0,0 +1,44 @@
+//go:build !windows
+// +build !windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"syscall"
+
+	"golang.org/x/sys/unix"
+
+	"k8s.io/klog/v2"
+)
+
+func permitPortReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEPORT on socket: %v", err)
+		}
+	})
+}
+
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		if err := syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEADDR, 1); err != nil {
+			klog.Warningf("failed to set SO_REUSEADDR on socket: %v", err)
+		}
+	})
+}

vendor/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -0,0 +1,35 @@
+//go:build windows
+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func permitPortReuse(network, address string, c syscall.RawConn) error {
+	return fmt.Errorf("port reuse is not supported on Windows")
+}
+
+// Windows supports SO_REUSEADDR, but it may cause undefined behavior, as
+// there is no protection against port hijacking.
+func permitAddressReuse(network, addr string, conn syscall.RawConn) error {
+	return fmt.Errorf("address reuse is not supported on Windows")
+}

vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"github.com/google/uuid"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
+)
+
+type SecureServingOptionsWithLoopback struct {
+	*SecureServingOptions
+}
+
+func (o *SecureServingOptions) WithLoopback() *SecureServingOptionsWithLoopback {
+	return &SecureServingOptionsWithLoopback{o}
+}
+
+// ApplyTo fills up serving information in the server configuration.
+func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.SecureServingInfo, loopbackClientConfig **rest.Config) error {
+	if s == nil || s.SecureServingOptions == nil || secureServingInfo == nil {
+		return nil
+	}
+
+	if err := s.SecureServingOptions.ApplyTo(secureServingInfo); err != nil {
+		return err
+	}
+
+	if *secureServingInfo == nil || loopbackClientConfig == nil {
+		return nil
+	}
+
+	// create self-signed cert+key with the fake server.LoopbackClientServerNameOverride and
+	// let the server return it when the loopback client connects.
+	certPem, keyPem, err := certutil.GenerateSelfSignedCertKey(server.LoopbackClientServerNameOverride, nil, nil)
+	if err != nil {
+		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
+	}
+	certProvider, err := dynamiccertificates.NewStaticSNICertKeyContent("self-signed loopback", certPem, keyPem, server.LoopbackClientServerNameOverride)
+	if err != nil {
+		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
+	}
+
+	// Write to the front of SNICerts so that this overrides any other certs with the same name
+	(*secureServingInfo).SNICerts = append([]dynamiccertificates.SNICertKeyContentProvider{certProvider}, (*secureServingInfo).SNICerts...)
+
+	secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.New().String(), certPem)
+	switch {
+	// if we failed and there's no fallback loopback client config, we need to fail
+	case err != nil && *loopbackClientConfig == nil:
+		(*secureServingInfo).SNICerts = (*secureServingInfo).SNICerts[1:]
+		return err
+
+	// if we failed, but we already have a fallback loopback client config (usually insecure), allow it
+	case err != nil && *loopbackClientConfig != nil:
+
+	default:
+		*loopbackClientConfig = secureLoopbackClientConfig
+	}
+
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/options/tracing.go

@@ -0,0 +1,162 @@
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+
+	"github.com/spf13/pflag"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/sdk/resource"
+	"go.opentelemetry.io/otel/semconv/v1.12.0"
+	"google.golang.org/grpc"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/feature"
+	tracing "k8s.io/component-base/tracing"
+	tracingapi "k8s.io/component-base/tracing/api/v1"
+	"k8s.io/utils/path"
+)
+
+const apiserverService = "apiserver"
+
+var (
+	cfgScheme = runtime.NewScheme()
+	codecs    = serializer.NewCodecFactory(cfgScheme)
+)
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// TracingOptions contain configuration options for tracing
+// exporters
+type TracingOptions struct {
+	// ConfigFile is the file path with api-server tracing configuration.
+	ConfigFile string
+}
+
+// NewTracingOptions creates a new instance of TracingOptions
+func NewTracingOptions() *TracingOptions {
+	return &TracingOptions{}
+}
+
+// AddFlags adds flags related to tracing to the specified FlagSet
+func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "tracing-config-file", o.ConfigFile,
+		"File with apiserver tracing configuration.")
+}
+
+// ApplyTo fills up Tracing config with options.
+func (o *TracingOptions) ApplyTo(es *egressselector.EgressSelector, c *server.Config) error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+	if !feature.DefaultFeatureGate.Enabled(features.APIServerTracing) {
+		return fmt.Errorf("APIServerTracing feature is not enabled, but tracing config file was provided")
+	}
+
+	traceConfig, err := ReadTracingConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read tracing config: %v", err)
+	}
+
+	errs := tracingapi.ValidateTracingConfiguration(traceConfig, feature.DefaultFeatureGate, nil)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate tracing configuration: %v", errs.ToAggregate())
+	}
+
+	opts := []otlptracegrpc.Option{}
+	if es != nil {
+		// Only use the egressselector dialer if egressselector is enabled.
+		// Endpoint is on the "ControlPlane" network
+		egressDialer, err := es.Lookup(egressselector.ControlPlane.AsNetworkContext())
+		if err != nil {
+			return err
+		}
+		if egressDialer != nil {
+			otelDialer := func(ctx context.Context, addr string) (net.Conn, error) {
+				return egressDialer(ctx, "tcp", addr)
+			}
+			opts = append(opts, otlptracegrpc.WithDialOption(grpc.WithContextDialer(otelDialer)))
+		}
+	}
+
+	resourceOpts := []resource.Option{
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String(apiserverService),
+			semconv.ServiceInstanceIDKey.String(c.APIServerID),
+		),
+	}
+	tp, err := tracing.NewProvider(context.Background(), traceConfig, opts, resourceOpts)
+	if err != nil {
+		return err
+	}
+	c.TracerProvider = tp
+	if c.LoopbackClientConfig != nil {
+		c.LoopbackClientConfig.Wrap(tracing.WrapperFor(c.TracerProvider))
+	}
+	return nil
+}
+
+// Validate verifies flags passed to TracingOptions.
+func (o *TracingOptions) Validate() (errs []error) {
+	if o == nil || o.ConfigFile == "" {
+		return
+	}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists {
+		errs = append(errs, fmt.Errorf("tracing-config-file %s does not exist", o.ConfigFile))
+	} else if err != nil {
+		errs = append(errs, fmt.Errorf("error checking if tracing-config-file %s exists: %v", o.ConfigFile, err))
+	}
+	return
+}
+
+// ReadTracingConfiguration reads the tracing configuration from a file
+func ReadTracingConfiguration(configFilePath string) (*tracingapi.TracingConfiguration, error) {
+	if configFilePath == "" {
+		return nil, fmt.Errorf("tracing config file was empty")
+	}
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read tracing configuration from %q: %v", configFilePath, err)
+	}
+	internalConfig := &apiserver.TracingConfiguration{}
+	// this handles json/yaml/whatever, and decodes all registered version to the internal version
+	if err := runtime.DecodeInto(codecs.UniversalDecoder(), data, internalConfig); err != nil {
+		return nil, fmt.Errorf("unable to decode tracing configuration data: %v", err)
+	}
+	tc := &tracingapi.TracingConfiguration{
+		Endpoint:               internalConfig.Endpoint,
+		SamplingRatePerMillion: internalConfig.SamplingRatePerMillion,
+	}
+	return tc, nil
+}

vendor/k8s.io/apiserver/pkg/server/plugins.go

@@ -0,0 +1,34 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+// This file exists to force the desired plugin implementations to be linked into genericapi pkg.
+import (
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle"
+	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy"
+	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
+	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
+)
+
+// RegisterAllAdmissionPlugins registers all admission plugins
+func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
+	lifecycle.Register(plugins)
+	validatingwebhook.Register(plugins)
+	mutatingwebhook.Register(plugins)
+	validatingadmissionpolicy.Register(plugins)
+}

vendor/k8s.io/apiserver/pkg/server/resourceconfig/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package resourceconfig contains the resource config related helper functions.
+package resourceconfig // import "k8s.io/apiserver/pkg/server/resourceconfig"

vendor/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resourceconfig
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	serverstore "k8s.io/apiserver/pkg/server/storage"
+	cliflag "k8s.io/component-base/cli/flag"
+)
+
+// GroupVersionRegistry provides access to registered group versions.
+type GroupVersionRegistry interface {
+	// IsGroupRegistered returns true if given group is registered.
+	IsGroupRegistered(group string) bool
+	// IsVersionRegistered returns true if given version is registered.
+	IsVersionRegistered(v schema.GroupVersion) bool
+	// PrioritizedVersionsAllGroups returns all registered group versions.
+	PrioritizedVersionsAllGroups() []schema.GroupVersion
+}
+
+// MergeResourceEncodingConfigs merges the given defaultResourceConfig with specific GroupVersionResource overrides.
+func MergeResourceEncodingConfigs(
+	defaultResourceEncoding *serverstore.DefaultResourceEncodingConfig,
+	resourceEncodingOverrides []schema.GroupVersionResource,
+) *serverstore.DefaultResourceEncodingConfig {
+	resourceEncodingConfig := defaultResourceEncoding
+	for _, gvr := range resourceEncodingOverrides {
+		resourceEncodingConfig.SetResourceEncoding(gvr.GroupResource(), gvr.GroupVersion(),
+			schema.GroupVersion{Group: gvr.Group, Version: runtime.APIVersionInternal})
+	}
+	return resourceEncodingConfig
+}
+
+// Recognized values for the --runtime-config parameter to enable/disable groups of APIs
+const (
+	APIAll   = "api/all"
+	APIGA    = "api/ga"
+	APIBeta  = "api/beta"
+	APIAlpha = "api/alpha"
+)
+
+var (
+	gaPattern    = regexp.MustCompile(`^v\d+$`)
+	betaPattern  = regexp.MustCompile(`^v\d+beta\d+$`)
+	alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
+
+	groupVersionMatchers = map[string]func(gv schema.GroupVersion) bool{
+		// allows users to address all api versions
+		APIAll: func(gv schema.GroupVersion) bool { return true },
+		// allows users to address all api versions in the form v[0-9]+
+		APIGA: func(gv schema.GroupVersion) bool { return gaPattern.MatchString(gv.Version) },
+		// allows users to address all beta api versions
+		APIBeta: func(gv schema.GroupVersion) bool { return betaPattern.MatchString(gv.Version) },
+		// allows users to address all alpha api versions
+		APIAlpha: func(gv schema.GroupVersion) bool { return alphaPattern.MatchString(gv.Version) },
+	}
+
+	groupVersionMatchersOrder = []string{APIAll, APIGA, APIBeta, APIAlpha}
+)
+
+// MergeAPIResourceConfigs merges the given defaultAPIResourceConfig with the given resourceConfigOverrides.
+// Exclude the groups not registered in registry, and check if version is
+// not registered in group, then it will fail.
+func MergeAPIResourceConfigs(
+	defaultAPIResourceConfig *serverstore.ResourceConfig,
+	resourceConfigOverrides cliflag.ConfigurationMap,
+	registry GroupVersionRegistry,
+) (*serverstore.ResourceConfig, error) {
+	resourceConfig := defaultAPIResourceConfig
+	overrides := resourceConfigOverrides
+
+	for _, flag := range groupVersionMatchersOrder {
+		if value, ok := overrides[flag]; ok {
+			if value == "false" {
+				resourceConfig.DisableMatchingVersions(groupVersionMatchers[flag])
+			} else if value == "true" {
+				resourceConfig.EnableMatchingVersions(groupVersionMatchers[flag])
+			} else {
+				return nil, fmt.Errorf("invalid value %v=%v", flag, value)
+			}
+		}
+	}
+
+	type versionEnablementPreference struct {
+		key          string
+		enabled      bool
+		groupVersion schema.GroupVersion
+	}
+	type resourceEnablementPreference struct {
+		key                  string
+		enabled              bool
+		groupVersionResource schema.GroupVersionResource
+	}
+	versionPreferences := []versionEnablementPreference{}
+	resourcePreferences := []resourceEnablementPreference{}
+
+	// "<resourceSpecifier>={true|false} allows users to enable/disable API.
+	// This takes preference over api/all, if specified.
+	// Iterate through all group/version overrides specified in runtimeConfig.
+	for key := range overrides {
+		// Have already handled them above. Can skip them here.
+		if _, ok := groupVersionMatchers[key]; ok {
+			continue
+		}
+
+		tokens := strings.Split(key, "/")
+		if len(tokens) < 2 || len(tokens) > 3 {
+			continue
+		}
+		groupVersionString := tokens[0] + "/" + tokens[1]
+		groupVersion, err := schema.ParseGroupVersion(groupVersionString)
+		if err != nil {
+			return nil, fmt.Errorf("invalid key %s", key)
+		}
+
+		// Exclude group not registered into the registry.
+		if !registry.IsGroupRegistered(groupVersion.Group) {
+			continue
+		}
+
+		// Verify that the groupVersion is registered into registry.
+		if !registry.IsVersionRegistered(groupVersion) {
+			return nil, fmt.Errorf("group version %s that has not been registered", groupVersion.String())
+		}
+		enabled, err := getRuntimeConfigValue(overrides, key, false)
+		if err != nil {
+			return nil, err
+		}
+
+		switch len(tokens) {
+		case 2:
+			versionPreferences = append(versionPreferences, versionEnablementPreference{
+				key:          key,
+				enabled:      enabled,
+				groupVersion: groupVersion,
+			})
+		case 3:
+			if strings.ToLower(tokens[2]) != tokens[2] {
+				return nil, fmt.Errorf("invalid key %v: group/version/resource and resource is always lowercase plural, not %q", key, tokens[2])
+			}
+			resourcePreferences = append(resourcePreferences, resourceEnablementPreference{
+				key:                  key,
+				enabled:              enabled,
+				groupVersionResource: groupVersion.WithResource(tokens[2]),
+			})
+		}
+	}
+
+	// apply version preferences first, so that we can remove the hardcoded resource preferences that are being overridden
+	for _, versionPreference := range versionPreferences {
+		if versionPreference.enabled {
+			// enable the groupVersion for "group/version=true"
+			resourceConfig.EnableVersions(versionPreference.groupVersion)
+
+		} else {
+			// disable the groupVersion only for "group/version=false"
+			resourceConfig.DisableVersions(versionPreference.groupVersion)
+		}
+	}
+
+	// apply resource preferences last, so they have the highest priority
+	for _, resourcePreference := range resourcePreferences {
+		if resourcePreference.enabled {
+			// enable the resource for "group/version/resource=true"
+			resourceConfig.EnableResources(resourcePreference.groupVersionResource)
+		} else {
+			resourceConfig.DisableResources(resourcePreference.groupVersionResource)
+		}
+	}
+
+	return resourceConfig, nil
+}
+
+func getRuntimeConfigValue(overrides cliflag.ConfigurationMap, apiKey string, defaultValue bool) (bool, error) {
+	flagValue, ok := overrides[apiKey]
+	if ok {
+		if flagValue == "" {
+			return true, nil
+		}
+		boolValue, err := strconv.ParseBool(flagValue)
+		if err != nil {
+			return false, fmt.Errorf("invalid value of %s: %s, err: %v", apiKey, flagValue, err)
+		}
+		return boolValue, nil
+	}
+	return defaultValue, nil
+}
+
+// ParseGroups takes in resourceConfig and returns parsed groups.
+func ParseGroups(resourceConfig cliflag.ConfigurationMap) ([]string, error) {
+	groups := []string{}
+	for key := range resourceConfig {
+		if _, ok := groupVersionMatchers[key]; ok {
+			continue
+		}
+		tokens := strings.Split(key, "/")
+		if len(tokens) != 2 && len(tokens) != 3 {
+			return groups, fmt.Errorf("runtime-config invalid key %s", key)
+		}
+		groupVersionString := tokens[0] + "/" + tokens[1]
+		groupVersion, err := schema.ParseGroupVersion(groupVersionString)
+		if err != nil {
+			return nil, fmt.Errorf("runtime-config invalid key %s", key)
+		}
+		groups = append(groups, groupVersion.Group)
+	}
+
+	return groups, nil
+}

vendor/k8s.io/apiserver/pkg/server/routes/OWNERS

@@ -0,0 +1,4 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+reviewers:
+  - sttts

vendor/k8s.io/apiserver/pkg/server/routes/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package routes holds a collection of optional genericapiserver http handlers.
+package routes // import "k8s.io/apiserver/pkg/server/routes"

vendor/k8s.io/apiserver/pkg/server/routes/flags.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"net/http"
+	"path"
+	"sync"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+var (
+	lock            = &sync.RWMutex{}
+	registeredFlags = map[string]debugFlag{}
+)
+
+// DebugFlags adds handlers for flags under /debug/flags.
+type DebugFlags struct {
+}
+
+// Install registers the APIServer's flags handler.
+func (f DebugFlags) Install(c *mux.PathRecorderMux, flag string, handler func(http.ResponseWriter, *http.Request)) {
+	c.UnlistedHandle("/debug/flags", http.HandlerFunc(f.Index))
+	c.UnlistedHandlePrefix("/debug/flags/", http.HandlerFunc(f.Index))
+
+	url := path.Join("/debug/flags", flag)
+	c.UnlistedHandleFunc(url, handler)
+
+	f.addFlag(flag)
+}
+
+// Index responds with the `/debug/flags` request.
+// For example, "/debug/flags/v" serves the "--v" flag.
+// Index responds to a request for "/debug/flags/" with an HTML page
+// listing the available flags.
+func (f DebugFlags) Index(w http.ResponseWriter, r *http.Request) {
+	lock.RLock()
+	defer lock.RUnlock()
+	if err := indexTmpl.Execute(w, registeredFlags); err != nil {
+		klog.Error(err)
+	}
+}
+
+var indexTmpl = template.Must(template.New("index").Parse(`<html>
+<head>
+<title>/debug/flags/</title>
+</head>
+<body>
+/debug/flags/<br>
+<br>
+flags:<br>
+<table>
+{{range .}}
+<tr>{{.Flag}}<br>
+{{end}}
+</table>
+<br>
+full flags configurable<br>
+</body>
+</html>
+`))
+
+type debugFlag struct {
+	Flag string
+}
+
+func (f DebugFlags) addFlag(flag string) {
+	lock.Lock()
+	defer lock.Unlock()
+	registeredFlags[flag] = debugFlag{flag}
+}
+
+// StringFlagSetterFunc is a func used for setting string type flag.
+type StringFlagSetterFunc func(string) (string, error)
+
+// StringFlagPutHandler wraps an http Handler to set string type flag.
+func StringFlagPutHandler(setter StringFlagSetterFunc) http.HandlerFunc {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		switch {
+		case req.Method == "PUT":
+			body, err := ioutil.ReadAll(req.Body)
+			if err != nil {
+				writePlainText(http.StatusBadRequest, "error reading request body: "+err.Error(), w)
+				return
+			}
+			defer req.Body.Close()
+			response, err := setter(string(body))
+			if err != nil {
+				writePlainText(http.StatusBadRequest, err.Error(), w)
+				return
+			}
+			writePlainText(http.StatusOK, response, w)
+			return
+		default:
+			writePlainText(http.StatusNotAcceptable, "unsupported http method", w)
+			return
+		}
+	})
+}
+
+// writePlainText renders a simple string response.
+func writePlainText(statusCode int, text string, w http.ResponseWriter) {
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(statusCode)
+	fmt.Fprintln(w, text)
+}

vendor/k8s.io/apiserver/pkg/server/routes/index.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// ListedPathProvider is an interface for providing paths that should be reported at /.
+type ListedPathProvider interface {
+	// ListedPaths is an alphabetically sorted list of paths to be reported at /.
+	ListedPaths() []string
+}
+
+// ListedPathProviders is a convenient way to combine multiple ListedPathProviders
+type ListedPathProviders []ListedPathProvider
+
+// ListedPaths unions and sorts the included paths.
+func (p ListedPathProviders) ListedPaths() []string {
+	ret := sets.String{}
+	for _, provider := range p {
+		for _, path := range provider.ListedPaths() {
+			ret.Insert(path)
+		}
+	}
+
+	return ret.List()
+}
+
+// Index provides a webservice for the http root / listing all known paths.
+type Index struct{}
+
+// Install adds the Index webservice to the given mux.
+func (i Index) Install(pathProvider ListedPathProvider, mux *mux.PathRecorderMux) {
+	handler := IndexLister{StatusCode: http.StatusOK, PathProvider: pathProvider}
+
+	mux.UnlistedHandle("/", handler)
+	mux.UnlistedHandle("/index.html", handler)
+}
+
+// IndexLister lists the available indexes with the status code provided
+type IndexLister struct {
+	StatusCode   int
+	PathProvider ListedPathProvider
+}
+
+// ServeHTTP serves the available paths.
+func (i IndexLister) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	responsewriters.WriteRawJSON(i.StatusCode, metav1.RootPaths{Paths: i.PathProvider.ListedPaths()}, w)
+}

vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -0,0 +1,53 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/mux"
+	cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics"
+	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
+	flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// DefaultMetrics installs the default prometheus metrics handler
+type DefaultMetrics struct{}
+
+// Install adds the DefaultMetrics handler
+func (m DefaultMetrics) Install(c *mux.PathRecorderMux) {
+	register()
+	c.Handle("/metrics", legacyregistry.Handler())
+}
+
+// MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method
+// which resets the metrics.
+type MetricsWithReset struct{}
+
+// Install adds the MetricsWithReset handler
+func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
+	register()
+	c.Handle("/metrics", legacyregistry.HandlerWithReset())
+}
+
+// register apiserver and etcd metrics
+func register() {
+	apimetrics.Register()
+	cachermetrics.Register()
+	etcd3metrics.Register()
+	flowcontrolmetrics.Register()
+}

vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	restful "github.com/emicklei/go-restful/v3"
+	"k8s.io/klog/v2"
+
+	"k8s.io/apiserver/pkg/server/mux"
+	builder2 "k8s.io/kube-openapi/pkg/builder"
+	"k8s.io/kube-openapi/pkg/builder3"
+	"k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/handler"
+	"k8s.io/kube-openapi/pkg/handler3"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+// OpenAPI installs spec endpoints for each web service.
+type OpenAPI struct {
+	Config *common.Config
+}
+
+// Install adds the SwaggerUI webservice to the given mux.
+func (oa OpenAPI) InstallV2(c *restful.Container, mux *mux.PathRecorderMux) (*handler.OpenAPIService, *spec.Swagger) {
+	spec, err := builder2.BuildOpenAPISpec(c.RegisteredWebServices(), oa.Config)
+	if err != nil {
+		klog.Fatalf("Failed to build open api spec for root: %v", err)
+	}
+	spec.Definitions = handler.PruneDefaults(spec.Definitions)
+	openAPIVersionedService, err := handler.NewOpenAPIService(spec)
+	if err != nil {
+		klog.Fatalf("Failed to create OpenAPIService: %v", err)
+	}
+
+	err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
+	if err != nil {
+		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
+	}
+
+	return openAPIVersionedService, spec
+}
+
+// InstallV3 adds the static group/versions defined in the RegisteredWebServices to the OpenAPI v3 spec
+func (oa OpenAPI) InstallV3(c *restful.Container, mux *mux.PathRecorderMux) *handler3.OpenAPIService {
+	openAPIVersionedService, err := handler3.NewOpenAPIService(nil)
+	if err != nil {
+		klog.Fatalf("Failed to create OpenAPIService: %v", err)
+	}
+
+	err = openAPIVersionedService.RegisterOpenAPIV3VersionedService("/openapi/v3", mux)
+	if err != nil {
+		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
+	}
+
+	grouped := make(map[string][]*restful.WebService)
+
+	for _, t := range c.RegisteredWebServices() {
+		// Strip the "/" prefix from the name
+		gvName := t.RootPath()[1:]
+		grouped[gvName] = []*restful.WebService{t}
+	}
+
+	for gv, ws := range grouped {
+		spec, err := builder3.BuildOpenAPISpec(ws, oa.Config)
+		if err != nil {
+			klog.Errorf("Failed to build OpenAPI v3 for group %s, %q", gv, err)
+
+		}
+		openAPIVersionedService.UpdateGroupVersion(gv, spec)
+	}
+	return openAPIVersionedService
+}

vendor/k8s.io/apiserver/pkg/server/routes/profiling.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+	"net/http/pprof"
+
+	"k8s.io/apiserver/pkg/server/mux"
+)
+
+// Profiling adds handlers for pprof under /debug/pprof.
+type Profiling struct{}
+
+// Install adds the Profiling webservice to the given mux.
+func (d Profiling) Install(c *mux.PathRecorderMux) {
+	c.UnlistedHandleFunc("/debug/pprof", redirectTo("/debug/pprof/"))
+	c.UnlistedHandlePrefix("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	c.UnlistedHandleFunc("/debug/pprof/profile", pprof.Profile)
+	c.UnlistedHandleFunc("/debug/pprof/symbol", pprof.Symbol)
+	c.UnlistedHandleFunc("/debug/pprof/trace", pprof.Trace)
+}
+
+// redirectTo redirects request to a certain destination.
+func redirectTo(to string) func(http.ResponseWriter, *http.Request) {
+	return func(rw http.ResponseWriter, req *http.Request) {
+		http.Redirect(rw, req, to, http.StatusFound)
+	}
+}

vendor/k8s.io/apiserver/pkg/server/routes/version.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful/v3"
+
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+)
+
+// Version provides a webservice with version information.
+type Version struct {
+	Version *version.Info
+}
+
+// Install registers the APIServer's `/version` handler.
+func (v Version) Install(c *restful.Container) {
+	if v.Version == nil {
+		return
+	}
+
+	// Set up a service to return the git code version.
+	versionWS := new(restful.WebService)
+	versionWS.Path("/version")
+	versionWS.Doc("git code version from which this is built")
+	versionWS.Route(
+		versionWS.GET("/").To(v.handleVersion).
+			Doc("get the code version").
+			Operation("getCodeVersion").
+			Produces(restful.MIME_JSON).
+			Consumes(restful.MIME_JSON).
+			Writes(version.Info{}))
+
+	c.Add(versionWS)
+}
+
+// handleVersion writes the server's version information.
+func (v Version) handleVersion(req *restful.Request, resp *restful.Response) {
+	responsewriters.WriteRawJSON(http.StatusOK, *v.Version, resp.ResponseWriter)
+}

vendor/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -0,0 +1,303 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/net/http2"
+	"k8s.io/component-base/cli/flag"
+	"k8s.io/klog/v2"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+)
+
+const (
+	defaultKeepAlivePeriod = 3 * time.Minute
+)
+
+// tlsConfig produces the tls.Config to serve with.
+func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, error) {
+	tlsConfig := &tls.Config{
+		// Can't use SSLv3 because of POODLE and BEAST
+		// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+		// Can't use TLSv1.1 because of RC4 cipher usage
+		MinVersion: tls.VersionTLS12,
+		// enable HTTP2 for go's 1.7 HTTP Server
+		NextProtos: []string{"h2", "http/1.1"},
+	}
+
+	// these are static aspects of the tls.Config
+	if s.DisableHTTP2 {
+		klog.Info("Forcing use of http/1.1 only")
+		tlsConfig.NextProtos = []string{"http/1.1"}
+	}
+	if s.MinTLSVersion > 0 {
+		tlsConfig.MinVersion = s.MinTLSVersion
+	}
+	if len(s.CipherSuites) > 0 {
+		tlsConfig.CipherSuites = s.CipherSuites
+		insecureCiphers := flag.InsecureTLSCiphers()
+		for i := 0; i < len(s.CipherSuites); i++ {
+			for cipherName, cipherID := range insecureCiphers {
+				if s.CipherSuites[i] == cipherID {
+					klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
+				}
+			}
+		}
+	}
+
+	if s.ClientCA != nil {
+		// Populate PeerCertificates in requests, but don't reject connections without certificates
+		// This allows certificates to be validated by authenticators, while still allowing other auth types
+		tlsConfig.ClientAuth = tls.RequestClientCert
+	}
+
+	if s.ClientCA != nil || s.Cert != nil || len(s.SNICerts) > 0 {
+		dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
+			tlsConfig,
+			s.ClientCA,
+			s.Cert,
+			s.SNICerts,
+			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
+		)
+
+		if s.ClientCA != nil {
+			s.ClientCA.AddListener(dynamicCertificateController)
+		}
+		if s.Cert != nil {
+			s.Cert.AddListener(dynamicCertificateController)
+		}
+		// generate a context from stopCh. This is to avoid modifying files which are relying on apiserver
+		// TODO: See if we can pass ctx to the current method
+		ctx, cancel := context.WithCancel(context.Background())
+		go func() {
+			select {
+			case <-stopCh:
+				cancel() // stopCh closed, so cancel our context
+			case <-ctx.Done():
+			}
+		}()
+		// start controllers if possible
+		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(ctx); err != nil {
+				klog.Warningf("Initial population of client CA failed: %v", err)
+			}
+
+			go controller.Run(ctx, 1)
+		}
+		if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(ctx); err != nil {
+				klog.Warningf("Initial population of default serving certificate failed: %v", err)
+			}
+
+			go controller.Run(ctx, 1)
+		}
+		for _, sniCert := range s.SNICerts {
+			sniCert.AddListener(dynamicCertificateController)
+			if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok {
+				// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+				// Files are required to be populated already, so this is for convenience.
+				if err := controller.RunOnce(ctx); err != nil {
+					klog.Warningf("Initial population of SNI serving certificate failed: %v", err)
+				}
+
+				go controller.Run(ctx, 1)
+			}
+		}
+
+		// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+		// Files are required to be populated already, so this is for convenience.
+		if err := dynamicCertificateController.RunOnce(); err != nil {
+			klog.Warningf("Initial population of dynamic certificates failed: %v", err)
+		}
+		go dynamicCertificateController.Run(1, stopCh)
+
+		tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
+	}
+
+	return tlsConfig, nil
+}
+
+// Serve runs the secure http server. It fails only if certificates cannot be loaded or the initial listen call fails.
+// The actual server loop (stoppable by closing stopCh) runs in a go routine, i.e. Serve does not block.
+// It returns a stoppedCh that is closed when all non-hijacked active requests have been processed.
+// It returns a listenerStoppedCh that is closed when the underlying http Server has stopped listening.
+func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) (<-chan struct{}, <-chan struct{}, error) {
+	if s.Listener == nil {
+		return nil, nil, fmt.Errorf("listener must not be nil")
+	}
+
+	tlsConfig, err := s.tlsConfig(stopCh)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	secureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+		TLSConfig:      tlsConfig,
+
+		IdleTimeout:       90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+		ReadHeaderTimeout: 32 * time.Second, // just shy of requestTimeoutUpperBound
+	}
+
+	// At least 99% of serialized resources in surveyed clusters were smaller than 256kb.
+	// This should be big enough to accommodate most API POST requests in a single frame,
+	// and small enough to allow a per connection buffer of this size multiplied by `MaxConcurrentStreams`.
+	const resourceBody99Percentile = 256 * 1024
+
+	http2Options := &http2.Server{
+		IdleTimeout: 90 * time.Second, // matches http.DefaultTransport keep-alive timeout
+	}
+
+	// shrink the per-stream buffer and max framesize from the 1MB default while still accommodating most API POST requests in a single frame
+	http2Options.MaxUploadBufferPerStream = resourceBody99Percentile
+	http2Options.MaxReadFrameSize = resourceBody99Percentile
+
+	// use the overridden concurrent streams setting or make the default of 250 explicit so we can size MaxUploadBufferPerConnection appropriately
+	if s.HTTP2MaxStreamsPerConnection > 0 {
+		http2Options.MaxConcurrentStreams = uint32(s.HTTP2MaxStreamsPerConnection)
+	} else {
+		http2Options.MaxConcurrentStreams = 250
+	}
+
+	// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
+	http2Options.MaxUploadBufferPerConnection = http2Options.MaxUploadBufferPerStream * int32(http2Options.MaxConcurrentStreams)
+
+	if !s.DisableHTTP2 {
+		// apply settings to the server
+		if err := http2.ConfigureServer(secureServer, http2Options); err != nil {
+			return nil, nil, fmt.Errorf("error configuring http2: %v", err)
+		}
+	}
+
+	// use tlsHandshakeErrorWriter to handle messages of tls handshake error
+	tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
+	tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
+	secureServer.ErrorLog = tlsErrorLogger
+
+	klog.Infof("Serving securely on %s", secureServer.Addr)
+	return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
+}
+
+// RunServer spawns a go-routine continuously serving until the stopCh is
+// closed.
+// It returns a stoppedCh that is closed when all non-hijacked active requests
+// have been processed.
+// This function does not block
+// TODO: make private when insecure serving is gone from the kube-apiserver
+func RunServer(
+	server *http.Server,
+	ln net.Listener,
+	shutDownTimeout time.Duration,
+	stopCh <-chan struct{},
+) (<-chan struct{}, <-chan struct{}, error) {
+	if ln == nil {
+		return nil, nil, fmt.Errorf("listener must not be nil")
+	}
+
+	// Shutdown server gracefully.
+	serverShutdownCh, listenerStoppedCh := make(chan struct{}), make(chan struct{})
+	go func() {
+		defer close(serverShutdownCh)
+		<-stopCh
+		ctx, cancel := context.WithTimeout(context.Background(), shutDownTimeout)
+		server.Shutdown(ctx)
+		cancel()
+	}()
+
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer close(listenerStoppedCh)
+
+		var listener net.Listener
+		listener = tcpKeepAliveListener{ln}
+		if server.TLSConfig != nil {
+			listener = tls.NewListener(listener, server.TLSConfig)
+		}
+
+		err := server.Serve(listener)
+
+		msg := fmt.Sprintf("Stopped listening on %s", ln.Addr().String())
+		select {
+		case <-stopCh:
+			klog.Info(msg)
+		default:
+			panic(fmt.Sprintf("%s due to error: %v", msg, err))
+		}
+	}()
+
+	return serverShutdownCh, listenerStoppedCh, nil
+}
+
+// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
+// connections. It's used by ListenAndServe and ListenAndServeTLS so
+// dead TCP connections (e.g. closing laptop mid-download) eventually
+// go away.
+//
+// Copied from Go 1.7.2 net/http/server.go
+type tcpKeepAliveListener struct {
+	net.Listener
+}
+
+func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
+	c, err := ln.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	if tc, ok := c.(*net.TCPConn); ok {
+		tc.SetKeepAlive(true)
+		tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
+	}
+	return c, nil
+}
+
+// tlsHandshakeErrorWriter writes TLS handshake errors to klog with
+// trace level - V(5), to avoid flooding of tls handshake errors.
+type tlsHandshakeErrorWriter struct {
+	out io.Writer
+}
+
+const tlsHandshakeErrorPrefix = "http: TLS handshake error"
+
+func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
+	if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
+		klog.V(5).Info(string(p))
+		metrics.TLSHandshakeErrors.Inc()
+		return len(p), nil
+	}
+
+	// for non tls handshake error, log it as usual
+	return w.out.Write(p)
+}

vendor/k8s.io/apiserver/pkg/server/signal.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"os"
+	"os/signal"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+var shutdownHandler chan os.Signal
+
+// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
+// which is closed on one of these signals. If a second signal is caught, the program
+// is terminated with exit code 1.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
+func SetupSignalHandler() <-chan struct{} {
+	return SetupSignalContext().Done()
+}
+
+// SetupSignalContext is same as SetupSignalHandler, but a context.Context is returned.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
+func SetupSignalContext() context.Context {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	shutdownHandler = make(chan os.Signal, 2)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	signal.Notify(shutdownHandler, shutdownSignals...)
+	go func() {
+		<-shutdownHandler
+		cancel()
+		<-shutdownHandler
+		os.Exit(1) // second signal. Exit directly.
+	}()
+
+	return ctx
+}
+
+// RequestShutdown emulates a received event that is considered as shutdown signal (SIGTERM/SIGINT)
+// This returns whether a handler was notified
+func RequestShutdown() bool {
+	if shutdownHandler != nil {
+		select {
+		case shutdownHandler <- shutdownSignals[0]:
+			return true
+		default:
+		}
+	}
+
+	return false
+}

vendor/k8s.io/apiserver/pkg/server/signal_posix.go

@@ -0,0 +1,27 @@
+//go:build !windows
+// +build !windows
+
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+	"syscall"
+)
+
+var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}

vendor/k8s.io/apiserver/pkg/server/signal_windows.go

@@ -0,0 +1,23 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"os"
+)
+
+var shutdownSignals = []os.Signal{os.Interrupt}

vendor/k8s.io/apiserver/pkg/server/storage/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package storage contains the plumbing to setup the etcd storage of the apiserver.
+package storage // import "k8s.io/apiserver/pkg/server/storage"

vendor/k8s.io/apiserver/pkg/server/storage/resource_config.go

@@ -0,0 +1,153 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// APIResourceConfigSource is the interface to determine which groups and versions are enabled
+type APIResourceConfigSource interface {
+	ResourceEnabled(resource schema.GroupVersionResource) bool
+	AnyResourceForGroupEnabled(group string) bool
+}
+
+var _ APIResourceConfigSource = &ResourceConfig{}
+
+type ResourceConfig struct {
+	GroupVersionConfigs map[schema.GroupVersion]bool
+	ResourceConfigs     map[schema.GroupVersionResource]bool
+}
+
+func NewResourceConfig() *ResourceConfig {
+	return &ResourceConfig{GroupVersionConfigs: map[schema.GroupVersion]bool{}, ResourceConfigs: map[schema.GroupVersionResource]bool{}}
+}
+
+// DisableMatchingVersions disables all group/versions for which the matcher function returns true.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) DisableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for version := range o.GroupVersionConfigs {
+		if matcher(version) {
+			o.GroupVersionConfigs[version] = false
+			o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+		}
+	}
+}
+
+// EnableMatchingVersions enables all group/versions for which the matcher function returns true.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) EnableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for version := range o.GroupVersionConfigs {
+		if matcher(version) {
+			o.GroupVersionConfigs[version] = true
+			o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+		}
+	}
+}
+
+// resourceMatcherForVersion matches resources in the specified version
+func resourceMatcherForVersion(gv schema.GroupVersion) func(gvr schema.GroupVersionResource) bool {
+	return func(gvr schema.GroupVersionResource) bool {
+		return gv == gvr.GroupVersion()
+	}
+}
+
+// removeMatchingResourcePreferences removes individual resource preferences that match.  This is useful when an override of a version or level enablement should
+// override the previously individual preferences.
+func (o *ResourceConfig) removeMatchingResourcePreferences(matcher func(gvr schema.GroupVersionResource) bool) {
+	keysToRemove := []schema.GroupVersionResource{}
+	for k := range o.ResourceConfigs {
+		if matcher(k) {
+			keysToRemove = append(keysToRemove, k)
+		}
+	}
+	for _, k := range keysToRemove {
+		delete(o.ResourceConfigs, k)
+	}
+}
+
+// DisableVersions disables the versions entirely.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) DisableVersions(versions ...schema.GroupVersion) {
+	for _, version := range versions {
+		o.GroupVersionConfigs[version] = false
+
+		// a preference about a version takes priority over the previously set resources
+		o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+	}
+}
+
+// EnableVersions enables all resources in a given groupVersion.
+// This will remove any preferences previously set on individual resources.
+func (o *ResourceConfig) EnableVersions(versions ...schema.GroupVersion) {
+	for _, version := range versions {
+		o.GroupVersionConfigs[version] = true
+
+		// a preference about a version takes priority over the previously set resources
+		o.removeMatchingResourcePreferences(resourceMatcherForVersion(version))
+	}
+
+}
+
+// TODO this must be removed and we enable/disable individual resources.
+func (o *ResourceConfig) versionEnabled(version schema.GroupVersion) bool {
+	enabled, _ := o.GroupVersionConfigs[version]
+	return enabled
+}
+
+func (o *ResourceConfig) DisableResources(resources ...schema.GroupVersionResource) {
+	for _, resource := range resources {
+		o.ResourceConfigs[resource] = false
+	}
+}
+
+func (o *ResourceConfig) EnableResources(resources ...schema.GroupVersionResource) {
+	for _, resource := range resources {
+		o.ResourceConfigs[resource] = true
+	}
+}
+
+func (o *ResourceConfig) ResourceEnabled(resource schema.GroupVersionResource) bool {
+	// if a resource is explicitly set, that takes priority over the preference of the version.
+	resourceEnabled, explicitlySet := o.ResourceConfigs[resource]
+	if explicitlySet {
+		return resourceEnabled
+	}
+
+	if !o.versionEnabled(resource.GroupVersion()) {
+		return false
+	}
+	// they are enabled by default.
+	return true
+}
+
+func (o *ResourceConfig) AnyResourceForGroupEnabled(group string) bool {
+	for version := range o.GroupVersionConfigs {
+		if version.Group == group {
+			if o.versionEnabled(version) {
+				return true
+			}
+		}
+	}
+	for resource := range o.ResourceConfigs {
+		if resource.Group == group && o.ResourceEnabled(resource) {
+			return true
+		}
+	}
+
+	return false
+}

vendor/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type ResourceEncodingConfig interface {
+	// StorageEncoding returns the serialization format for the resource.
+	// TODO this should actually return a GroupVersionKind since you can logically have multiple "matching" Kinds
+	// For now, it returns just the GroupVersion for consistency with old behavior
+	StorageEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
+
+	// InMemoryEncodingFor returns the groupVersion for the in memory representation the storage should convert to.
+	InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
+}
+
+type DefaultResourceEncodingConfig struct {
+	// resources records the overriding encoding configs for individual resources.
+	resources map[schema.GroupResource]*OverridingResourceEncoding
+	scheme    *runtime.Scheme
+}
+
+type OverridingResourceEncoding struct {
+	ExternalResourceEncoding schema.GroupVersion
+	InternalResourceEncoding schema.GroupVersion
+}
+
+var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
+
+func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
+	return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
+}
+
+func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
+	o.resources[resourceBeingStored] = &OverridingResourceEncoding{
+		ExternalResourceEncoding: externalEncodingVersion,
+		InternalResourceEncoding: internalVersion,
+	}
+}
+
+func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.ExternalResourceEncoding, nil
+	}
+
+	// return the most preferred external version for the group
+	return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
+}
+
+func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.InternalResourceEncoding, nil
+	}
+	return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
+}

vendor/k8s.io/apiserver/pkg/server/storage/storage_codec.go

@@ -0,0 +1,103 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"fmt"
+	"mime"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+)
+
+// StorageCodecConfig are the arguments passed to newStorageCodecFn
+type StorageCodecConfig struct {
+	StorageMediaType  string
+	StorageSerializer runtime.StorageSerializer
+	StorageVersion    schema.GroupVersion
+	MemoryVersion     schema.GroupVersion
+	Config            storagebackend.Config
+
+	EncoderDecoratorFn func(runtime.Encoder) runtime.Encoder
+	DecoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
+}
+
+// NewStorageCodec assembles a storage codec for the provided storage media type, the provided serializer, and the requested
+// storage and memory versions.
+func NewStorageCodec(opts StorageCodecConfig) (runtime.Codec, runtime.GroupVersioner, error) {
+	mediaType, _, err := mime.ParseMediaType(opts.StorageMediaType)
+	if err != nil {
+		return nil, nil, fmt.Errorf("%q is not a valid mime-type", opts.StorageMediaType)
+	}
+
+	supportedMediaTypes := opts.StorageSerializer.SupportedMediaTypes()
+	serializer, ok := runtime.SerializerInfoForMediaType(supportedMediaTypes, mediaType)
+	if !ok {
+		supportedMediaTypeList := make([]string, len(supportedMediaTypes))
+		for i, mediaType := range supportedMediaTypes {
+			supportedMediaTypeList[i] = mediaType.MediaType
+		}
+		return nil, nil, fmt.Errorf("unable to find serializer for %q, supported media types: %v", mediaType, supportedMediaTypeList)
+	}
+
+	s := serializer.Serializer
+
+	// Give callers the opportunity to wrap encoders and decoders.  For decoders, each returned decoder will
+	// be passed to the recognizer so that multiple decoders are available.
+	var encoder runtime.Encoder = s
+	if opts.EncoderDecoratorFn != nil {
+		encoder = opts.EncoderDecoratorFn(encoder)
+	}
+	decoders := []runtime.Decoder{
+		// selected decoder as the primary
+		s,
+		// universal deserializer as a fallback
+		opts.StorageSerializer.UniversalDeserializer(),
+		// base64-wrapped universal deserializer as a last resort.
+		// this allows reading base64-encoded protobuf, which should only exist if etcd2+protobuf was used at some point.
+		// data written that way could exist in etcd2, or could have been migrated to etcd3.
+		// TODO: flag this type of data if we encounter it, require migration (read to decode, write to persist using a supported encoder), and remove in 1.8
+		runtime.NewBase64Serializer(nil, opts.StorageSerializer.UniversalDeserializer()),
+	}
+	if opts.DecoderDecoratorFn != nil {
+		decoders = opts.DecoderDecoratorFn(decoders)
+	}
+
+	encodeVersioner := runtime.NewMultiGroupVersioner(
+		opts.StorageVersion,
+		schema.GroupKind{Group: opts.StorageVersion.Group},
+		schema.GroupKind{Group: opts.MemoryVersion.Group},
+	)
+
+	// Ensure the storage receives the correct version.
+	encoder = opts.StorageSerializer.EncoderForVersion(
+		encoder,
+		encodeVersioner,
+	)
+	decoder := opts.StorageSerializer.DecoderToVersion(
+		recognizer.NewDecoder(decoders...),
+		runtime.NewCoercingMultiGroupVersioner(
+			opts.MemoryVersion,
+			schema.GroupKind{Group: opts.MemoryVersion.Group},
+			schema.GroupKind{Group: opts.StorageVersion.Group},
+		),
+	)
+
+	return runtime.NewCodec(encoder, decoder), encodeVersioner, nil
+}

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -0,0 +1,349 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storage
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"io/ioutil"
+	"strings"
+
+	"k8s.io/klog/v2"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/storage/storagebackend"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+)
+
+// Backend describes the storage servers, the information here should be enough
+// for health validations.
+type Backend struct {
+	// the url of storage backend like: https://etcd.domain:2379
+	Server string
+	// the required tls config
+	TLSConfig *tls.Config
+}
+
+// StorageFactory is the interface to locate the storage for a given GroupResource
+type StorageFactory interface {
+	// New finds the storage destination for the given group and resource. It will
+	// return an error if the group has no storage destination configured.
+	NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
+
+	// ResourcePrefix returns the overridden resource prefix for the GroupResource
+	// This allows for cohabitation of resources with different native types and provides
+	// centralized control over the shape of etcd directories
+	ResourcePrefix(groupResource schema.GroupResource) string
+
+	// Backends gets all backends for all registered storage destinations.
+	// Used for getting all instances for health validations.
+	Backends() []Backend
+}
+
+// DefaultStorageFactory takes a GroupResource and returns back its storage interface.  This result includes:
+// 1. Merged etcd config, including: auth, server locations, prefixes
+// 2. Resource encodings for storage: group,version,kind to store as
+// 3. Cohabitating default: some resources like hpa are exposed through multiple APIs.  They must agree on 1 and 2
+type DefaultStorageFactory struct {
+	// StorageConfig describes how to create a storage backend in general.
+	// Its authentication information will be used for every storage.Interface returned.
+	StorageConfig storagebackend.Config
+
+	Overrides map[schema.GroupResource]groupResourceOverrides
+
+	DefaultResourcePrefixes map[schema.GroupResource]string
+
+	// DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used.
+	DefaultMediaType string
+
+	// DefaultSerializer is used to create encoders and decoders for the storage.Interface.
+	DefaultSerializer runtime.StorageSerializer
+
+	// ResourceEncodingConfig describes how to encode a particular GroupVersionResource
+	ResourceEncodingConfig ResourceEncodingConfig
+
+	// APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API
+	// This is discrete from resource enablement because those are separate concerns.  How this source is configured
+	// is left to the caller.
+	APIResourceConfigSource APIResourceConfigSource
+
+	// newStorageCodecFn exists to be overwritten for unit testing.
+	newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, encodeVersioner runtime.GroupVersioner, err error)
+}
+
+type groupResourceOverrides struct {
+	// etcdLocation contains the list of "special" locations that are used for particular GroupResources
+	// These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource
+	etcdLocation []string
+	// etcdPrefix is the base location for a GroupResource.
+	etcdPrefix string
+	// etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location
+	// If empty, the default mapping is used.  If the default mapping doesn't contain an entry, it will use
+	// the ToLowered name of the resource, not including the group.
+	etcdResourcePrefix string
+	// mediaType is the desired serializer to choose. If empty, the default is chosen.
+	mediaType string
+	// serializer contains the list of "special" serializers for a GroupResource.  Resource=* means for the entire group
+	serializer runtime.StorageSerializer
+	// cohabitatingResources keeps track of which resources must be stored together.  This happens when we have multiple ways
+	// of exposing one set of concepts.  autoscaling.HPA and extensions.HPA as a for instance
+	// The order of the slice matters!  It is the priority order of lookup for finding a storage location
+	cohabitatingResources []schema.GroupResource
+	// encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized.
+	encoderDecoratorFn func(runtime.Encoder) runtime.Encoder
+	// decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of
+	// returned decoders will be priority for attempt to decode.
+	decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
+	// disablePaging will prevent paging on the provided resource.
+	disablePaging bool
+}
+
+// Apply overrides the provided config and options if the override has a value in that position
+func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) {
+	if len(o.etcdLocation) > 0 {
+		config.Transport.ServerList = o.etcdLocation
+	}
+	if len(o.etcdPrefix) > 0 {
+		config.Prefix = o.etcdPrefix
+	}
+
+	if len(o.mediaType) > 0 {
+		options.StorageMediaType = o.mediaType
+	}
+	if o.serializer != nil {
+		options.StorageSerializer = o.serializer
+	}
+	if o.encoderDecoratorFn != nil {
+		options.EncoderDecoratorFn = o.encoderDecoratorFn
+	}
+	if o.decoderDecoratorFn != nil {
+		options.DecoderDecoratorFn = o.decoderDecoratorFn
+	}
+	if o.disablePaging {
+		config.Paging = false
+	}
+}
+
+var _ StorageFactory = &DefaultStorageFactory{}
+
+const AllResources = "*"
+
+func NewDefaultStorageFactory(
+	config storagebackend.Config,
+	defaultMediaType string,
+	defaultSerializer runtime.StorageSerializer,
+	resourceEncodingConfig ResourceEncodingConfig,
+	resourceConfig APIResourceConfigSource,
+	specialDefaultResourcePrefixes map[schema.GroupResource]string,
+) *DefaultStorageFactory {
+	config.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
+	if len(defaultMediaType) == 0 {
+		defaultMediaType = runtime.ContentTypeJSON
+	}
+	return &DefaultStorageFactory{
+		StorageConfig:           config,
+		Overrides:               map[schema.GroupResource]groupResourceOverrides{},
+		DefaultMediaType:        defaultMediaType,
+		DefaultSerializer:       defaultSerializer,
+		ResourceEncodingConfig:  resourceEncodingConfig,
+		APIResourceConfigSource: resourceConfig,
+		DefaultResourcePrefixes: specialDefaultResourcePrefixes,
+
+		newStorageCodecFn: NewStorageCodec,
+	}
+}
+
+func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdLocation = location
+	s.Overrides[groupResource] = overrides
+}
+
+func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdPrefix = prefix
+	s.Overrides[groupResource] = overrides
+}
+
+// SetDisableAPIListChunking allows a specific resource to disable paging at the storage layer, to prevent
+// exposure of key names in continuations. This may be overridden by feature gates.
+func (s *DefaultStorageFactory) SetDisableAPIListChunking(groupResource schema.GroupResource) {
+	overrides := s.Overrides[groupResource]
+	overrides.disablePaging = true
+	s.Overrides[groupResource] = overrides
+}
+
+// SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir.  You'll end up in `etcdPrefix/resourceEtcdPrefix`.
+func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) {
+	overrides := s.Overrides[groupResource]
+	overrides.etcdResourcePrefix = prefix
+	s.Overrides[groupResource] = overrides
+}
+
+func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) {
+	overrides := s.Overrides[groupResource]
+	overrides.mediaType = mediaType
+	overrides.serializer = serializer
+	s.Overrides[groupResource] = overrides
+}
+
+// AddCohabitatingResources links resources together the order of the slice matters!  its the priority order of lookup for finding a storage location
+func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) {
+	for _, groupResource := range groupResources {
+		overrides := s.Overrides[groupResource]
+		overrides.cohabitatingResources = groupResources
+		s.Overrides[groupResource] = overrides
+	}
+}
+
+func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) {
+	for _, groupResource := range groupResources {
+		overrides := s.Overrides[groupResource]
+		overrides.encoderDecoratorFn = encoderDecoratorFn
+		overrides.decoderDecoratorFn = decoderDecoratorFn
+		s.Overrides[groupResource] = overrides
+	}
+}
+
+func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource {
+	return schema.GroupResource{Group: resource.Group, Resource: AllResources}
+}
+
+func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource {
+	for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources {
+		// TODO deads2k or liggitt determine if have ever stored any of our cohabitating resources in a different location on new clusters
+		if s.APIResourceConfigSource.AnyResourceForGroupEnabled(potentialStorageResource.Group) {
+			return potentialStorageResource
+		}
+	}
+
+	return groupResource
+}
+
+// New finds the storage destination for the given group and resource. It will
+// return an error if the group has no storage destination configured.
+func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+	chosenStorageResource := s.getStorageGroupResource(groupResource)
+
+	// operate on copy
+	storageConfig := s.StorageConfig
+	codecConfig := StorageCodecConfig{
+		StorageMediaType:  s.DefaultMediaType,
+		StorageSerializer: s.DefaultSerializer,
+	}
+
+	if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok {
+		override.Apply(&storageConfig, &codecConfig)
+	}
+	if override, ok := s.Overrides[chosenStorageResource]; ok {
+		override.Apply(&storageConfig, &codecConfig)
+	}
+
+	var err error
+	codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
+	if err != nil {
+		return nil, err
+	}
+	codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
+	if err != nil {
+		return nil, err
+	}
+	codecConfig.Config = storageConfig
+
+	storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)
+	if err != nil {
+		return nil, err
+	}
+	klog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config)
+
+	return storageConfig.ForResource(groupResource), nil
+}
+
+// Backends returns all backends for all registered storage destinations.
+// Used for getting all instances for health validations.
+func (s *DefaultStorageFactory) Backends() []Backend {
+	return backends(s.StorageConfig, s.Overrides)
+}
+
+// Backends returns all backends for all registered storage destinations.
+// Used for getting all instances for health validations.
+func Backends(storageConfig storagebackend.Config) []Backend {
+	return backends(storageConfig, nil)
+}
+
+func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []Backend {
+	servers := sets.NewString(storageConfig.Transport.ServerList...)
+
+	for _, overrides := range grOverrides {
+		servers.Insert(overrides.etcdLocation...)
+	}
+
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	if len(storageConfig.Transport.CertFile) > 0 && len(storageConfig.Transport.KeyFile) > 0 {
+		cert, err := tls.LoadX509KeyPair(storageConfig.Transport.CertFile, storageConfig.Transport.KeyFile)
+		if err != nil {
+			klog.Errorf("failed to load key pair while getting backends: %s", err)
+		} else {
+			tlsConfig.Certificates = []tls.Certificate{cert}
+		}
+	}
+	if len(storageConfig.Transport.TrustedCAFile) > 0 {
+		if caCert, err := ioutil.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
+			klog.Errorf("failed to read ca file while getting backends: %s", err)
+		} else {
+			caPool := x509.NewCertPool()
+			caPool.AppendCertsFromPEM(caCert)
+			tlsConfig.RootCAs = caPool
+			tlsConfig.InsecureSkipVerify = false
+		}
+	}
+
+	backends := []Backend{}
+	for server := range servers {
+		backends = append(backends, Backend{
+			Server: server,
+			// We can't share TLSConfig across different backends to avoid races.
+			// For more details see: https://pr.k8s.io/59338
+			TLSConfig: tlsConfig.Clone(),
+		})
+	}
+	return backends
+}
+
+func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string {
+	chosenStorageResource := s.getStorageGroupResource(groupResource)
+	groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)]
+	exactResourceOverride := s.Overrides[chosenStorageResource]
+
+	etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource]
+	if len(groupOverride.etcdResourcePrefix) > 0 {
+		etcdResourcePrefix = groupOverride.etcdResourcePrefix
+	}
+	if len(exactResourceOverride.etcdResourcePrefix) > 0 {
+		etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix
+	}
+	if len(etcdResourcePrefix) == 0 {
+		etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource)
+	}
+
+	return etcdResourcePrefix
+}