rebase: update kubernetes to latest

updating the kubernetes release to the
latest in main go.mod

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -32,9 +32,9 @@ import (
 	"sync/atomic"
 	"time"
 
-	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/google/uuid"
 	"golang.org/x/crypto/cryptobyte"
+	jsonpatch "gopkg.in/evanphx/json-patch.v4"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -42,8 +42,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/version"
 	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
-	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
@@ -64,14 +64,17 @@ import (
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
+	"k8s.io/apiserver/pkg/server/routine"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
 	storagevalue "k8s.io/apiserver/pkg/storage/value"
 	"k8s.io/apiserver/pkg/storageversion"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
 	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/component-base/featuregate"
 	"k8s.io/component-base/logs"
 	"k8s.io/component-base/metrics/features"
 	"k8s.io/component-base/metrics/prometheus/slis"
@@ -147,8 +150,11 @@ type Config struct {
 	// done values in this values for this map are ignored.
 	PostStartHooks map[string]PostStartHookConfigEntry
 
-	// Version will enable the /version endpoint if non-nil
-	Version *version.Info
+	// EffectiveVersion determines which apis and features are available
+	// based on when the api/feature lifecyle.
+	EffectiveVersion utilversion.EffectiveVersion
+	// FeatureGate is a way to plumb feature gate through if you have them.
+	FeatureGate featuregate.FeatureGate
 	// AuditBackend is where audit events are sent to.
 	AuditBackend audit.Backend
 	// AuditPolicyRuleEvaluator makes the decision of whether and how to audit log a request.
@@ -211,6 +217,10 @@ type Config struct {
 	// twice this value.  Note that it is up to the request handlers to ignore or honor this timeout. In seconds.
 	MinRequestTimeout int
 
+	// StorageInitializationTimeout defines the maximum amount of time to wait for storage initialization
+	// before declaring apiserver ready.
+	StorageInitializationTimeout time.Duration
+
 	// This represents the maximum amount of time it should take for apiserver to complete its startup
 	// sequence and become healthy. From apiserver's start time to when this amount of time has
 	// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and
@@ -421,6 +431,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 		MaxMutatingRequestsInFlight:    200,
 		RequestTimeout:                 time.Duration(60) * time.Second,
 		MinRequestTimeout:              1800,
+		StorageInitializationTimeout:   time.Minute,
 		LivezGracePeriod:               time.Duration(0),
 		ShutdownDelayDuration:          time.Duration(0),
 		// 1.5MB is the default client request size in bytes
@@ -585,7 +596,7 @@ func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
 	}
 }
 
-func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
+func completeOpenAPI(config *openapicommon.Config, version *version.Version) {
 	if config == nil {
 		return
 	}
@@ -624,7 +635,7 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
 	}
 }
 
-func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
+func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Version) {
 	if config == nil {
 		return
 	}
@@ -676,6 +687,9 @@ func (c *Config) ShutdownInitiatedNotify() <-chan struct{} {
 // Complete fills in any fields not set that are required to have valid data and can be derived
 // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
 func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
+	if c.FeatureGate == nil {
+		c.FeatureGate = utilfeature.DefaultFeatureGate
+	}
 	if len(c.ExternalAddress) == 0 && c.PublicAddress != nil {
 		c.ExternalAddress = c.PublicAddress.String()
 	}
@@ -691,9 +705,8 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
 		}
 		c.ExternalAddress = net.JoinHostPort(c.ExternalAddress, strconv.Itoa(port))
 	}
-
-	completeOpenAPI(c.OpenAPIConfig, c.Version)
-	completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
+	completeOpenAPI(c.OpenAPIConfig, c.EffectiveVersion.EmulationVersion())
+	completeOpenAPIV3(c.OpenAPIV3Config, c.EffectiveVersion.EmulationVersion())
 
 	if c.DiscoveryAddresses == nil {
 		c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@@ -711,7 +724,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
 		} else {
 			c.EquivalentResourceRegistry = runtime.NewEquivalentResourceRegistryWithIdentity(func(groupResource schema.GroupResource) string {
 				// use the storage prefix as the key if possible
-				if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource); err == nil {
+				if opts, err := c.RESTOptionsGetter.GetRESTOptions(groupResource, nil); err == nil {
 					return opts.ResourcePrefix
 				}
 				// otherwise return "" to use the default key (parent GV name)
@@ -817,14 +830,16 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		ShutdownSendRetryAfter: c.ShutdownSendRetryAfter,
 
 		APIServerID:           c.APIServerID,
+		StorageReadinessHook:  NewStorageReadinessHook(c.StorageInitializationTimeout),
 		StorageVersionManager: c.StorageVersionManager,
 
-		Version: c.Version,
+		EffectiveVersion: c.EffectiveVersion,
+		FeatureGate:      c.FeatureGate,
 
 		muxAndDiscoveryCompleteSignals: map[string]<-chan struct{}{},
 	}
 
-	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+	if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
 		manager := c.AggregatedDiscoveryGroupManager
 		if manager == nil {
 			manager = discoveryendpoint.NewResourceManager("apis")
@@ -1039,15 +1054,15 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 		handler = genericfilters.WithRetryAfter(handler, c.lifecycleSignals.NotAcceptingNewRequest.Signaled())
 	}
 	handler = genericfilters.WithHTTPLogging(handler)
-	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) {
+	if c.FeatureGate.Enabled(genericfeatures.APIServerTracing) {
 		handler = genericapifilters.WithTracing(handler, c.TracerProvider)
 	}
 	handler = genericapifilters.WithLatencyTrackers(handler)
 	// WithRoutine will execute future handlers in a separate goroutine and serving
 	// handler in current goroutine to minimize the stack memory usage. It must be
 	// after WithPanicRecover() to be protected from panics.
-	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
-		handler = genericfilters.WithRoutine(handler, c.LongRunningFunc)
+	if c.FeatureGate.Enabled(genericfeatures.APIServingWithRoutine) {
+		handler = routine.WithRoutine(handler, c.LongRunningFunc)
 	}
 	handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver)
 	handler = genericapifilters.WithRequestReceivedTimestamp(handler)
@@ -1087,10 +1102,10 @@ func installAPI(s *GenericAPIServer, c *Config) {
 		}
 	}
 
-	routes.Version{Version: c.Version}.Install(s.Handler.GoRestfulContainer)
+	routes.Version{Version: c.EffectiveVersion.BinaryVersion().Info()}.Install(s.Handler.GoRestfulContainer)
 
 	if c.EnableDiscovery {
-		if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
+		if c.FeatureGate.Enabled(genericfeatures.AggregatedDiscoveryEndpoint) {
 			wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(s.DiscoveryGroupManager, s.AggregatedDiscoveryGroupManager)
 			s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/apis", metav1.APIGroupList{}))
 		} else {

vendor/k8s.io/apiserver/pkg/server/deleted_kinds.go

@@ -17,6 +17,7 @@ limitations under the License.
 package server
 
 import (
+	"fmt"
 	"os"
 	"strconv"
 	"strings"
@@ -25,16 +26,15 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
-	apimachineryversion "k8s.io/apimachinery/pkg/version"
+	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/klog/v2"
 )
 
 // resourceExpirationEvaluator holds info for deciding if a particular rest.Storage needs to excluded from the API
 type resourceExpirationEvaluator struct {
-	currentMajor int
-	currentMinor int
-	isAlpha      bool
+	currentVersion *apimachineryversion.Version
+	isAlpha        bool
 	// This is usually set for testing for which tests need to be removed.  This prevent insta-failing CI.
 	// Set KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA to see what will be removed when we tag beta
 	strictRemovedHandlingInAlpha bool
@@ -53,30 +53,17 @@ type ResourceExpirationEvaluator interface {
 	ShouldServeForVersion(majorRemoved, minorRemoved int) bool
 }
 
-func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) {
+func NewResourceExpirationEvaluator(currentVersion *apimachineryversion.Version) (ResourceExpirationEvaluator, error) {
+	if currentVersion == nil {
+		return nil, fmt.Errorf("empty NewResourceExpirationEvaluator currentVersion")
+	}
+	klog.V(1).Infof("NewResourceExpirationEvaluator with currentVersion: %s.", currentVersion)
 	ret := &resourceExpirationEvaluator{
 		strictRemovedHandlingInAlpha: false,
 	}
-	if len(currentVersion.Major) > 0 {
-		currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32)
-		if err != nil {
-			return nil, err
-		}
-		ret.currentMajor = int(currentMajor64)
-	}
-	if len(currentVersion.Minor) > 0 {
-		// split the "normal" + and - for semver stuff
-		minorString := strings.Split(currentVersion.Minor, "+")[0]
-		minorString = strings.Split(minorString, "-")[0]
-		minorString = strings.Split(minorString, ".")[0]
-		currentMinor64, err := strconv.ParseInt(minorString, 10, 32)
-		if err != nil {
-			return nil, err
-		}
-		ret.currentMinor = int(currentMinor64)
-	}
-
-	ret.isAlpha = strings.Contains(currentVersion.GitVersion, "alpha")
+	// Only keeps the major and minor versions from input version.
+	ret.currentVersion = apimachineryversion.MajorMinor(currentVersion.Major(), currentVersion.Minor())
+	ret.isAlpha = strings.Contains(currentVersion.PreRelease(), "alpha")
 
 	if envString, ok := os.LookupEnv("KUBE_APISERVER_STRICT_REMOVED_API_HANDLING_IN_ALPHA"); !ok {
 		// do nothing
@@ -112,6 +99,15 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
 		return false
 	}
 
+	introduced, ok := versionedPtr.(introducedInterface)
+	if ok {
+		majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+		verIntroduced := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+		if e.currentVersion.LessThan(verIntroduced) {
+			return false
+		}
+	}
+
 	removed, ok := versionedPtr.(removedInterface)
 	if !ok {
 		return true
@@ -121,16 +117,11 @@ func (e *resourceExpirationEvaluator) shouldServe(gv schema.GroupVersion, versio
 }
 
 func (e *resourceExpirationEvaluator) ShouldServeForVersion(majorRemoved, minorRemoved int) bool {
-	if e.currentMajor < majorRemoved {
+	removedVer := apimachineryversion.MajorMinor(uint(majorRemoved), uint(minorRemoved))
+	if removedVer.GreaterThan(e.currentVersion) {
 		return true
 	}
-	if e.currentMajor > majorRemoved {
-		return false
-	}
-	if e.currentMinor < minorRemoved {
-		return true
-	}
-	if e.currentMinor > minorRemoved {
+	if removedVer.LessThan(e.currentVersion) {
 		return false
 	}
 	// at this point major and minor are equal, so this API should be removed when the current release GAs.
@@ -152,6 +143,11 @@ type removedInterface interface {
 	APILifecycleRemoved() (major, minor int)
 }
 
+// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
+type introducedInterface interface {
+	APILifecycleIntroduced() (major, minor int)
+}
+
 // removeDeletedKinds inspects the storage map and modifies it in place by removing storage for kinds that have been deleted.
 // versionedResourcesStorageMap mirrors the field on APIGroupInfo, it's a map from version to resource to the storage.
 func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versioner runtime.ObjectVersioner, versionedResourcesStorageMap map[string]map[string]rest.Storage) {
@@ -171,6 +167,8 @@ func (e *resourceExpirationEvaluator) RemoveDeletedKinds(groupName string, versi
 			}
 
 			klog.V(1).Infof("Removing resource %v.%v.%v because it is time to stop serving it per APILifecycle.", resourceName, apiVersion, groupName)
+			storage := versionToResource[resourceName]
+			storage.Destroy()
 			delete(versionToResource, resourceName)
 		}
 		versionedResourcesStorageMap[apiVersion] = versionToResource

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -54,7 +54,7 @@ type ConfigMapCAController struct {
 
 	listeners []Listener
 
-	queue workqueue.RateLimitingInterface
+	queue workqueue.TypedRateLimitingInterface[string]
 	// preRunCaches are the caches to sync before starting the work of this control loop
 	preRunCaches []cache.InformerSynced
 }
@@ -94,7 +94,10 @@ func NewDynamicCAFromConfigMapController(purpose, namespace, name, key string, k
 		configmapLister:    configmapLister,
 		configMapInformer:  uncastConfigmapInformer,
 
-		queue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)),
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)},
+		),
 		preRunCaches: []cache.InformerSynced{uncastConfigmapInformer.HasSynced},
 	}
 

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -60,7 +60,7 @@ type DynamicFileCAContent struct {
 	listeners []Listener
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
-	queue workqueue.RateLimitingInterface
+	queue workqueue.TypedRateLimitingInterface[string]
 }
 
 var _ Notifier = &DynamicFileCAContent{}
@@ -82,7 +82,10 @@ func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAConten
 	ret := &DynamicFileCAContent{
 		name:     name,
 		filename: filename,
-		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)},
+		),
 	}
 	if err := ret.loadCABundle(); err != nil {
 		return nil, err

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -47,7 +47,7 @@ type DynamicCertKeyPairContent struct {
 	listeners []Listener
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
-	queue workqueue.RateLimitingInterface
+	queue workqueue.TypedRateLimitingInterface[string]
 }
 
 var _ CertKeyContentProvider = &DynamicCertKeyPairContent{}
@@ -64,7 +64,10 @@ func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*Dyna
 		name:     name,
 		certFile: certFile,
 		keyFile:  keyFile,
-		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: fmt.Sprintf("DynamicCABundle-%s", purpose)},
+		),
 	}
 	if err := ret.loadCertKeyPair(); err != nil {
 		return nil, err

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -56,7 +56,7 @@ type DynamicServingCertificateController struct {
 	currentServingTLSConfig atomic.Value
 
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
-	queue         workqueue.RateLimitingInterface
+	queue         workqueue.TypedRateLimitingInterface[string]
 	eventRecorder events.EventRecorder
 }
 
@@ -76,7 +76,10 @@ func NewDynamicServingCertificateController(
 		servingCert:   servingCert,
 		sniCerts:      sniCerts,
 
-		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: "DynamicServingCertificateController"},
+		),
 		eventRecorder: eventRecorder,
 	}
 

vendor/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -137,9 +137,7 @@ func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 				}
 
 				metrics.RecordRequestPostTimeout(metrics.PostTimeoutSourceTimeoutHandler, status)
-				err := fmt.Errorf("post-timeout activity - time-elapsed: %s, %v %q result: %v",
-					time.Since(timedOutAt), r.Method, r.URL.Path, res)
-				utilruntime.HandleError(err)
+				utilruntime.HandleErrorWithContext(r.Context(), nil, "Post-timeout activity", "timeElapsed", time.Since(timedOutAt), "method", r.Method, "path", r.URL.Path, "result", res)
 			}()
 		}()
 		httplog.SetStacktracePredicate(r.Context(), func(status int) bool {

vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -17,7 +17,6 @@ limitations under the License.
 package filters
 
 import (
-	"fmt"
 	"net/http"
 
 	"k8s.io/apimachinery/pkg/util/runtime"
@@ -51,7 +50,7 @@ func WithPanicRecovery(handler http.Handler, resolver request.RequestInfoResolve
 			// This call can have different handlers, but the default chain rate limits. Call it after the metrics are updated
 			// in case the rate limit delays it.  If you outrun the rate for this one timed out requests, something has gone
 			// seriously wrong with your server, but generally having a logging signal for timeouts is useful.
-			runtime.HandleError(fmt.Errorf("timeout or abort while handling: method=%v URI=%q audit-ID=%q", req.Method, req.RequestURI, audit.GetAuditIDTruncated(req.Context())))
+			runtime.HandleErrorWithContext(req.Context(), nil, "Timeout or abort while handling", "method", req.Method, "URI", req.RequestURI, "auditID", audit.GetAuditIDTruncated(req.Context()))
 			return
 		}
 		http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -38,8 +38,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/managedfields"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/wait"
 	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
-	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -51,8 +51,9 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
 	"k8s.io/apiserver/pkg/storageversion"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/component-base/featuregate"
 	"k8s.io/klog/v2"
 	openapibuilder3 "k8s.io/kube-openapi/pkg/builder3"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
@@ -232,11 +233,18 @@ type GenericAPIServer struct {
 	// APIServerID is the ID of this API server
 	APIServerID string
 
+	// StorageReadinessHook implements post-start-hook functionality for checking readiness
+	// of underlying storage for registered resources.
+	StorageReadinessHook *StorageReadinessHook
+
 	// StorageVersionManager holds the storage versions of the API resources installed by this server.
 	StorageVersionManager storageversion.Manager
 
-	// Version will enable the /version endpoint if non-nil
-	Version *version.Info
+	// EffectiveVersion determines which apis and features are available
+	// based on when the api/feature lifecyle.
+	EffectiveVersion utilversion.EffectiveVersion
+	// FeatureGate is a way to plumb feature gate through if you have them.
+	FeatureGate featuregate.FeatureGate
 
 	// lifecycleSignals provides access to the various signals that happen during the life cycle of the apiserver.
 	lifecycleSignals lifecycleSignals
@@ -442,9 +450,19 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 
 // Run spawns the secure http server. It only returns if stopCh is closed
 // or the secure port cannot be listened on initially.
-// This is the diagram of what channels/signals are dependent on each other:
 //
-// |                                  stopCh
+// Deprecated: use RunWithContext instead. Run will not get removed to avoid
+// breaking consumers, but should not be used in new code.
+func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
+	ctx := wait.ContextForChannel(stopCh)
+	return s.RunWithContext(ctx)
+}
+
+// RunWithContext spawns the secure http server. It only returns if ctx is canceled
+// or the secure port cannot be listened on initially.
+// This is the diagram of what contexts/channels/signals are dependent on each other:
+//
+// |                                   ctx
 // |                                    |
 // |           ---------------------------------------------------------
 // |           |                                                       |
@@ -477,12 +495,13 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 // |           |                                         |                                        |
 // |           |-------------------|---------------------|----------------------------------------|
 // |                               |                     |
-// |                       stopHttpServerCh     (AuditBackend::Shutdown())
+// |                       stopHttpServerCtx     (AuditBackend::Shutdown())
 // |                               |
 // |                       listenerStoppedCh
 // |                               |
 // |      HTTPServerStoppedListening (httpServerStoppedListeningCh)
-func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
+func (s preparedGenericAPIServer) RunWithContext(ctx context.Context) error {
+	stopCh := ctx.Done()
 	delayedStopCh := s.lifecycleSignals.AfterShutdownDelayDuration
 	shutdownInitiatedCh := s.lifecycleSignals.ShutdownInitiated
 
@@ -544,9 +563,11 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 
 	notAcceptingNewRequestCh := s.lifecycleSignals.NotAcceptingNewRequest
 	drainedCh := s.lifecycleSignals.InFlightRequestsDrained
-	stopHttpServerCh := make(chan struct{})
+	// Canceling the parent context does not immediately cancel the HTTP server.
+	// We only inherit context values here and deal with cancellation ourselves.
+	stopHTTPServerCtx, stopHTTPServer := context.WithCancelCause(context.WithoutCancel(ctx))
 	go func() {
-		defer close(stopHttpServerCh)
+		defer stopHTTPServer(errors.New("time to stop HTTP server"))
 
 		timeToStopHttpServerCh := notAcceptingNewRequestCh.Signaled()
 		if s.ShutdownSendRetryAfter {
@@ -565,7 +586,7 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 		}
 	}
 
-	stoppedCh, listenerStoppedCh, err := s.NonBlockingRun(stopHttpServerCh, shutdownTimeout)
+	stoppedCh, listenerStoppedCh, err := s.NonBlockingRunWithContext(stopHTTPServerCtx, shutdownTimeout)
 	if err != nil {
 		return err
 	}
@@ -694,7 +715,18 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 // NonBlockingRun spawns the secure http server. An error is
 // returned if the secure port cannot be listened on.
 // The returned channel is closed when the (asynchronous) termination is finished.
+//
+// Deprecated: use RunWithContext instead. Run will not get removed to avoid
+// breaking consumers, but should not be used in new code.
 func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
+	ctx := wait.ContextForChannel(stopCh)
+	return s.NonBlockingRunWithContext(ctx, shutdownTimeout)
+}
+
+// NonBlockingRunWithContext spawns the secure http server. An error is
+// returned if the secure port cannot be listened on.
+// The returned channel is closed when the (asynchronous) termination is finished.
+func (s preparedGenericAPIServer) NonBlockingRunWithContext(ctx context.Context, shutdownTimeout time.Duration) (<-chan struct{}, <-chan struct{}, error) {
 	// Use an internal stop channel to allow cleanup of the listeners on error.
 	internalStopCh := make(chan struct{})
 	var stoppedCh <-chan struct{}
@@ -712,11 +744,11 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdow
 	// responsibility of the caller to close the provided channel to
 	// ensure cleanup.
 	go func() {
-		<-stopCh
+		<-ctx.Done()
 		close(internalStopCh)
 	}()
 
-	s.RunPostStartHooks(stopCh)
+	s.RunPostStartHooks(ctx)
 
 	if _, err := systemd.SdNotify(true, "READY=1\n"); err != nil {
 		klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
@@ -751,7 +783,7 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
 		}
 		resourceInfos = append(resourceInfos, r...)
 
-		if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
+		if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
 			// Aggregated discovery only aggregates resources under /apis
 			if apiPrefix == APIGroupPrefix {
 				s.AggregatedDiscoveryGroupManager.AddGroupVersion(
@@ -779,8 +811,8 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
 
 	s.RegisterDestroyFunc(apiGroupInfo.destroyStorage)
 
-	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
-		utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
+	if s.FeatureGate.Enabled(features.StorageVersionAPI) &&
+		s.FeatureGate.Enabled(features.APIServerIdentity) {
 		// API installation happens before we start listening on the handlers,
 		// therefore it is safe to register ResourceInfos here. The handler will block
 		// write requests until the storage versions of the targeting resources are updated.
@@ -810,12 +842,13 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
 	// Install the version handler.
 	// Add a handler at /<apiPrefix> to enumerate the supported api versions.
 	legacyRootAPIHandler := discovery.NewLegacyRootAPIHandler(s.discoveryAddresses, s.Serializer, apiPrefix)
-	if utilfeature.DefaultFeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
+	if s.FeatureGate.Enabled(features.AggregatedDiscoveryEndpoint) {
 		wrapped := discoveryendpoint.WrapAggregatedDiscoveryToHandler(legacyRootAPIHandler, s.AggregatedLegacyDiscoveryGroupManager)
 		s.Handler.GoRestfulContainer.Add(wrapped.GenerateWebService("/api", metav1.APIVersions{}))
 	} else {
 		s.Handler.GoRestfulContainer.Add(legacyRootAPIHandler.WebService())
 	}
+	s.registerStorageReadinessCheck("", apiGroupInfo)
 
 	return nil
 }
@@ -874,10 +907,28 @@ func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) erro
 
 		s.DiscoveryGroupManager.AddGroup(apiGroup)
 		s.Handler.GoRestfulContainer.Add(discovery.NewAPIGroupHandler(s.Serializer, apiGroup).WebService())
+		s.registerStorageReadinessCheck(apiGroupInfo.PrioritizedVersions[0].Group, apiGroupInfo)
 	}
 	return nil
 }
 
+// registerStorageReadinessCheck registers the readiness checks for all underlying storages
+// for a given APIGroup.
+func (s *GenericAPIServer) registerStorageReadinessCheck(groupName string, apiGroupInfo *APIGroupInfo) {
+	for version, storageMap := range apiGroupInfo.VersionedResourcesStorageMap {
+		for resource, storage := range storageMap {
+			if withReadiness, ok := storage.(rest.StorageWithReadiness); ok {
+				gvr := metav1.GroupVersionResource{
+					Group:    groupName,
+					Version:  version,
+					Resource: resource,
+				}
+				s.StorageReadinessHook.RegisterStorage(gvr, withReadiness)
+			}
+		}
+	}
+}
+
 // InstallAPIGroup exposes the given api group in the API.
 // The <apiGroupInfo> passed into this function shouldn't be used elsewhere as the
 // underlying storage will be destroyed on this servers shutdown.

vendor/k8s.io/apiserver/pkg/server/healthz.go

@@ -118,7 +118,7 @@ func (s *GenericAPIServer) AddLivezChecks(delay time.Duration, checks ...healthz
 // that we can register that the api-server is no longer ready while we attempt to gracefully
 // shutdown.
 func (s *GenericAPIServer) addReadyzShutdownCheck(stopCh <-chan struct{}) error {
-	return s.AddReadyzChecks(shutdownCheck{stopCh})
+	return s.AddReadyzChecks(healthz.NewShutdownHealthz(stopCh))
 }
 
 // installHealthz creates the healthz endpoint for this server
@@ -139,25 +139,6 @@ func (s *GenericAPIServer) installLivez() {
 	s.livezRegistry.installHandler(s.Handler.NonGoRestfulMux)
 }
 
-// shutdownCheck fails if the embedded channel is closed. This is intended to allow for graceful shutdown sequences
-// for the apiserver.
-type shutdownCheck struct {
-	StopCh <-chan struct{}
-}
-
-func (shutdownCheck) Name() string {
-	return "shutdown"
-}
-
-func (c shutdownCheck) Check(req *http.Request) error {
-	select {
-	case <-c.StopCh:
-		return fmt.Errorf("process is shutting down")
-	default:
-	}
-	return nil
-}
-
 // delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed. This
 // is intended for use primarily for livez health checks.
 func delayedHealthCheck(check healthz.HealthChecker, clock clock.Clock, delay time.Duration) healthz.HealthChecker {

vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -105,6 +105,29 @@ func (i *informerSync) Name() string {
 	return "informer-sync"
 }
 
+type shutdown struct {
+	stopCh <-chan struct{}
+}
+
+// NewShutdownHealthz returns a new HealthChecker that will fail if the embedded channel is closed.
+// This is intended to allow for graceful shutdown sequences.
+func NewShutdownHealthz(stopCh <-chan struct{}) HealthChecker {
+	return &shutdown{stopCh}
+}
+
+func (s *shutdown) Name() string {
+	return "shutdown"
+}
+
+func (s *shutdown) Check(req *http.Request) error {
+	select {
+	case <-s.stopCh:
+		return fmt.Errorf("process is shutting down")
+	default:
+	}
+	return nil
+}
+
 func (i *informerSync) Check(_ *http.Request) error {
 	stopCh := make(chan struct{})
 	// Close stopCh to force checking if informers are synced now.

vendor/k8s.io/apiserver/pkg/server/hooks.go

@@ -17,6 +17,7 @@ limitations under the License.
 package server
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -48,8 +49,13 @@ type PreShutdownHookFunc func() error
 type PostStartHookContext struct {
 	// LoopbackClientConfig is a config for a privileged loopback connection to the API server
 	LoopbackClientConfig *restclient.Config
-	// StopCh is the channel that will be closed when the server stops
+	// StopCh is the channel that will be closed when the server stops.
+	//
+	// Deprecated: use the PostStartHookContext itself instead, it contains a context that
+	// gets cancelled when the server stops. StopCh keeps getting provided for existing code.
 	StopCh <-chan struct{}
+	// Context gets cancelled when the server stops.
+	context.Context
 }
 
 // PostStartHookProvider is an interface in addition to provide a post start hook for the api server
@@ -151,15 +157,16 @@ func (s *GenericAPIServer) AddPreShutdownHookOrDie(name string, hook PreShutdown
 	}
 }
 
-// RunPostStartHooks runs the PostStartHooks for the server
-func (s *GenericAPIServer) RunPostStartHooks(stopCh <-chan struct{}) {
+// RunPostStartHooks runs the PostStartHooks for the server.
+func (s *GenericAPIServer) RunPostStartHooks(ctx context.Context) {
 	s.postStartHookLock.Lock()
 	defer s.postStartHookLock.Unlock()
 	s.postStartHooksCalled = true
 
 	context := PostStartHookContext{
 		LoopbackClientConfig: s.LoopbackClientConfig,
-		StopCh:               stopCh,
+		StopCh:               ctx.Done(),
+		Context:              ctx,
 	}
 
 	for hookName, hookEntry := range s.postStartHooks {

vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	"k8s.io/apiserver/pkg/server/routine"
 	"k8s.io/klog/v2"
 )
 
@@ -125,10 +126,26 @@ func withLogging(handler http.Handler, stackTracePred StacktracePred, shouldLogR
 		rl := newLoggedWithStartTime(req, w, startTime)
 		rl.StacktraceWhen(stackTracePred)
 		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
-		defer rl.Log()
+
+		var logFunc func()
+		logFunc = rl.Log
+		defer func() {
+			if logFunc != nil {
+				logFunc()
+			}
+		}()
 
 		w = responsewriter.WrapForHTTP1Or2(rl)
 		handler.ServeHTTP(w, req)
+
+		// We need to ensure that the request is logged after it is processed.
+		// In case the request is executed in a separate goroutine created via
+		// WithRoutine handler in the handler chain (i.e. above handler.ServeHTTP()
+		// would return request is completely responsed), we want the logging to
+		// happen in that goroutine too, so we append it to the task.
+		if routine.AppendTask(ctx, &routine.Task{Func: rl.Log}) {
+			logFunc = nil
+		}
 	})
 }
 

vendor/k8s.io/apiserver/pkg/server/mux/pathrecorder.go

@@ -96,9 +96,11 @@ func NewPathRecorderMux(name string) *PathRecorderMux {
 
 // ListedPaths returns the registered handler exposedPaths.
 func (m *PathRecorderMux) ListedPaths() []string {
+	m.lock.Lock()
 	handledPaths := append([]string{}, m.exposedPaths...)
-	sort.Strings(handledPaths)
+	m.lock.Unlock()
 
+	sort.Strings(handledPaths)
 	return handledPaths
 }
 

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -59,7 +59,7 @@ type AdmissionOptions struct {
 	// RecommendedPluginOrder holds an ordered list of plugin names we recommend to use by default
 	RecommendedPluginOrder []string
 	// DefaultOffPlugins is a set of plugin names that is disabled by default
-	DefaultOffPlugins sets.String
+	DefaultOffPlugins sets.Set[string]
 
 	// EnablePlugins indicates plugins to be enabled passed through `--enable-admission-plugins`.
 	EnablePlugins []string
@@ -91,7 +91,7 @@ func NewAdmissionOptions() *AdmissionOptions {
 		// after all the mutating ones, so their relative order in this list
 		// doesn't matter.
 		RecommendedPluginOrder: []string{lifecycle.PluginName, mutatingwebhook.PluginName, validatingadmissionpolicy.PluginName, validatingwebhook.PluginName},
-		DefaultOffPlugins:      sets.NewString(),
+		DefaultOffPlugins:      sets.Set[string]{},
 	}
 	server.RegisterAllAdmissionPlugins(options.Plugins)
 	return options
@@ -139,6 +139,9 @@ func (a *AdmissionOptions) ApplyTo(
 	if informers == nil {
 		return fmt.Errorf("admission depends on a Kubernetes core API shared informer, it cannot be nil")
 	}
+	if kubeClient == nil || dynamicClient == nil {
+		return fmt.Errorf("admission depends on a Kubernetes core API client, it cannot be nil")
+	}
 
 	pluginNames := a.enabledPluginNames()
 
@@ -223,7 +226,7 @@ func (a *AdmissionOptions) Validate() []error {
 // EnablePlugins, DisablePlugins fields
 // to prepare a list of ordered plugin names that are enabled.
 func (a *AdmissionOptions) enabledPluginNames() []string {
-	allOffPlugins := append(a.DefaultOffPlugins.List(), a.DisablePlugins...)
+	allOffPlugins := append(sets.List[string](a.DefaultOffPlugins), a.DisablePlugins...)
 	disabledPlugins := sets.NewString(allOffPlugins...)
 	enabledPlugins := sets.NewString(a.EnablePlugins...)
 	disabledPlugins = disabledPlugins.Difference(enabledPlugins)

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -26,6 +26,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server"
@@ -222,8 +223,8 @@ type DelegatingAuthenticationOptions struct {
 	// CustomRoundTripperFn allows for specifying a middleware function for custom HTTP behaviour for the authentication webhook client.
 	CustomRoundTripperFn transport.WrapperFunc
 
-	// DisableAnonymous gives user an option to disable Anonymous authentication.
-	DisableAnonymous bool
+	// Anonymous gives user an option to enable/disable Anonymous authentication.
+	Anonymous *apiserver.AnonymousAuthConfig
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -238,6 +239,7 @@ func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
 		},
 		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
 		TokenRequestTimeout: 10 * time.Second,
+		Anonymous:           &apiserver.AnonymousAuthConfig{Enabled: true},
 	}
 }
 
@@ -305,7 +307,7 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
 	}
 
 	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous:                !s.DisableAnonymous,
+		Anonymous:                &apiserver.AnonymousAuthConfig{Enabled: true},
 		CacheTTL:                 s.CacheTTL,
 		WebhookRetryBackoff:      s.WebhookRetryBackoff,
 		TokenAccessReviewTimeout: s.TokenRequestTimeout,

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -1,126 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package options
-
-import (
-	"fmt"
-	"net"
-
-	"github.com/spf13/pflag"
-
-	"k8s.io/apiserver/pkg/server"
-)
-
-// DeprecatedInsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
-// No one should be using these anymore.
-// DEPRECATED: all insecure serving options are removed in a future version
-type DeprecatedInsecureServingOptions struct {
-	BindAddress net.IP
-	BindPort    int
-	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
-	// "tcp4", and "tcp6".
-	BindNetwork string
-
-	// Listener is the secure server network listener.
-	// either Listener or BindAddress/BindPort/BindNetwork is set,
-	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
-	Listener net.Listener
-
-	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
-	// It defaults to options.CreateListener.
-	ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
-}
-
-// Validate ensures that the insecure port values within the range of the port.
-func (s *DeprecatedInsecureServingOptions) Validate() []error {
-	if s == nil {
-		return nil
-	}
-
-	errors := []error{}
-
-	if s.BindPort < 0 || s.BindPort > 65535 {
-		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
-	}
-
-	return errors
-}
-
-// AddFlags adds flags related to insecure serving to the specified FlagSet.
-func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
-	if s == nil {
-		return
-	}
-
-	fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
-		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening on all interfaces and IP address families).")
-	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
-	fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
-	fs.Lookup("insecure-bind-address").Hidden = false
-
-	fs.IntVar(&s.BindPort, "insecure-port", s.BindPort, ""+
-		"The port on which to serve unsecured, unauthenticated access.")
-	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
-	fs.MarkDeprecated("insecure-port", "This flag will be removed in a future version.")
-	fs.Lookup("insecure-port").Hidden = false
-}
-
-// AddUnqualifiedFlags adds flags related to insecure serving without the --insecure prefix to the specified FlagSet.
-func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet) {
-	if s == nil {
-		return
-	}
-
-	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
-		"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families).")
-	fs.MarkDeprecated("address", "see --bind-address instead.")
-	fs.Lookup("address").Hidden = false
-
-	fs.IntVar(&s.BindPort, "port", s.BindPort, "The port on which to serve unsecured, unauthenticated access. Set to 0 to disable.")
-	fs.MarkDeprecated("port", "see --secure-port instead.")
-	fs.Lookup("port").Hidden = false
-}
-
-// ApplyTo adds DeprecatedInsecureServingOptions to the insecureserverinfo and kube-controller manager configuration.
-// Note: the double pointer allows to set the *DeprecatedInsecureServingInfo to nil without referencing the struct hosting this pointer.
-func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecureServingInfo) error {
-	if s == nil {
-		return nil
-	}
-	if s.BindPort <= 0 {
-		return nil
-	}
-
-	if s.Listener == nil {
-		var err error
-		listen := CreateListener
-		if s.ListenFunc != nil {
-			listen = s.ListenFunc
-		}
-		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
-		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
-		if err != nil {
-			return fmt.Errorf("failed to create listener: %v", err)
-		}
-	}
-
-	*c = &server.DeprecatedInsecureServingInfo{
-		Listener: s.Listener,
-	}
-
-	return nil
-}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -130,7 +130,7 @@ func init() {
 	configScheme := runtime.NewScheme()
 	utilruntime.Must(apiserver.AddToScheme(configScheme))
 	utilruntime.Must(apiserverv1.AddToScheme(configScheme))
-	codecs = serializer.NewCodecFactory(configScheme)
+	codecs = serializer.NewCodecFactory(configScheme, serializer.EnableStrict)
 	envelopemetrics.RegisterMetrics()
 	storagevalue.RegisterMetrics()
 	metrics.RegisterMetrics()

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -49,7 +49,7 @@ type DynamicEncryptionConfigContent struct {
 	lastLoadedEncryptionConfigHash string
 
 	// queue for processing changes in encryption config file.
-	queue workqueue.RateLimitingInterface
+	queue workqueue.TypedRateLimitingInterface[string]
 
 	// dynamicTransformers updates the transformers when encryption config file changes.
 	dynamicTransformers *encryptionconfig.DynamicTransformers
@@ -78,8 +78,11 @@ func NewDynamicEncryptionConfiguration(
 		filePath:                       filePath,
 		lastLoadedEncryptionConfigHash: configContentHash,
 		dynamicTransformers:            dynamicTransformers,
-		queue:                          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
-		apiServerID:                    apiServerID,
+		queue: workqueue.NewTypedRateLimitingQueueWithConfig(
+			workqueue.DefaultTypedControllerRateLimiter[string](),
+			workqueue.TypedRateLimitingQueueConfig[string]{Name: name},
+		),
+		apiServerID: apiServerID,
 		getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
 			return encryptionconfig.GetEncryptionConfigHash(filepath)
 		},
@@ -150,7 +153,7 @@ func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.C
 	return true
 }
 
-func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
+func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey string) {
 	var (
 		updatedEffectiveConfig  bool
 		err                     error

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -383,8 +383,8 @@ type StorageFactoryRestOptionsFactory struct {
 	StorageFactory serverstorage.StorageFactory
 }
 
-func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource) (generic.RESTOptions, error) {
-	storageConfig, err := f.StorageFactory.NewConfig(resource)
+func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupResource, example runtime.Object) (generic.RESTOptions, error) {
+	storageConfig, err := f.StorageFactory.NewConfig(resource, example)
 	if err != nil {
 		return generic.RESTOptions{}, fmt.Errorf("unable to find storage destination for %v, due to %v", resource, err.Error())
 	}
@@ -399,6 +399,10 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
 		StorageObjectCountTracker: f.Options.StorageConfig.StorageObjectCountTracker,
 	}
 
+	if ret.StorageObjectCountTracker == nil {
+		ret.StorageObjectCountTracker = storageConfig.StorageObjectCountTracker
+	}
+
 	if f.Options.EnableWatchCache {
 		sizes, err := ParseWatchCacheSizes(f.Options.WatchCacheSizes)
 		if err != nil {
@@ -465,7 +469,7 @@ type SimpleStorageFactory struct {
 	StorageConfig storagebackend.Config
 }
 
-func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+func (s *SimpleStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
 	return s.StorageConfig.ForResource(resource), nil
 }
 
@@ -489,8 +493,8 @@ type transformerStorageFactory struct {
 	resourceTransformers storagevalue.ResourceTransformers
 }
 
-func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
-	config, err := t.delegate.NewConfig(resource)
+func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
+	config, err := t.delegate.NewConfig(resource, example)
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -71,6 +71,9 @@ func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interfac
 	c.EnableContentionProfiling = o.EnableContentionProfiling
 
 	if o.EnablePriorityAndFairness {
+		if clientset == nil {
+			return fmt.Errorf("invalid configuration: priority and fairness requires a core Kubernetes client")
+		}
 		if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
 			return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
 

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -120,9 +120,18 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
-	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
-	if err != nil {
-		return err
+	var kubeClient *kubernetes.Clientset
+	var dynamicClient *dynamic.DynamicClient
+	if config.ClientConfig != nil {
+		var err error
+		kubeClient, err = kubernetes.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
+		dynamicClient, err = dynamic.NewForConfig(config.ClientConfig)
+		if err != nil {
+			return err
+		}
 	}
 	if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
 		return err
@@ -131,10 +140,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err != nil {
 		return err
 	}
-	dynamicClient, err := dynamic.NewForConfig(config.ClientConfig)
-	if err != nil {
-		return err
-	}
 	if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, kubeClient, dynamicClient, o.FeatureGate,
 		initializers...); err != nil {
 		return err

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -25,8 +25,10 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/errors"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	utilversion "k8s.io/apiserver/pkg/util/version"
 
 	"github.com/spf13/pflag"
 )
@@ -45,16 +47,17 @@ const (
 type ServerRunOptions struct {
 	AdvertiseAddress net.IP
 
-	CorsAllowedOriginList       []string
-	HSTSDirectives              []string
-	ExternalHost                string
-	MaxRequestsInFlight         int
-	MaxMutatingRequestsInFlight int
-	RequestTimeout              time.Duration
-	GoawayChance                float64
-	LivezGracePeriod            time.Duration
-	MinRequestTimeout           int
-	ShutdownDelayDuration       time.Duration
+	CorsAllowedOriginList        []string
+	HSTSDirectives               []string
+	ExternalHost                 string
+	MaxRequestsInFlight          int
+	MaxMutatingRequestsInFlight  int
+	RequestTimeout               time.Duration
+	GoawayChance                 float64
+	LivezGracePeriod             time.Duration
+	MinRequestTimeout            int
+	StorageInitializationTimeout time.Duration
+	ShutdownDelayDuration        time.Duration
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
 	JSONPatchMaxCopyBytes int64
@@ -89,9 +92,24 @@ type ServerRunOptions struct {
 	// This grace period is orthogonal to other grace periods, and
 	// it is not overridden by any other grace period.
 	ShutdownWatchTerminationGracePeriod time.Duration
+
+	// ComponentGlobalsRegistry is the registry where the effective versions and feature gates for all components are stored.
+	ComponentGlobalsRegistry utilversion.ComponentGlobalsRegistry
+	// ComponentName is name under which the server's global variabled are registered in the ComponentGlobalsRegistry.
+	ComponentName string
 }
 
 func NewServerRunOptions() *ServerRunOptions {
+	if utilversion.DefaultComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent) == nil {
+		featureGate := utilfeature.DefaultMutableFeatureGate
+		effectiveVersion := utilversion.DefaultKubeEffectiveVersion()
+		utilruntime.Must(utilversion.DefaultComponentGlobalsRegistry.Register(utilversion.DefaultKubeComponent, effectiveVersion, featureGate))
+	}
+
+	return NewServerRunOptionsForComponent(utilversion.DefaultKubeComponent, utilversion.DefaultComponentGlobalsRegistry)
+}
+
+func NewServerRunOptionsForComponent(componentName string, componentGlobalsRegistry utilversion.ComponentGlobalsRegistry) *ServerRunOptions {
 	defaults := server.NewConfig(serializer.CodecFactory{})
 	return &ServerRunOptions{
 		MaxRequestsInFlight:                 defaults.MaxRequestsInFlight,
@@ -99,16 +117,22 @@ func NewServerRunOptions() *ServerRunOptions {
 		RequestTimeout:                      defaults.RequestTimeout,
 		LivezGracePeriod:                    defaults.LivezGracePeriod,
 		MinRequestTimeout:                   defaults.MinRequestTimeout,
+		StorageInitializationTimeout:        defaults.StorageInitializationTimeout,
 		ShutdownDelayDuration:               defaults.ShutdownDelayDuration,
 		ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
 		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
 		ShutdownSendRetryAfter:              false,
+		ComponentName:                       componentName,
+		ComponentGlobalsRegistry:            componentGlobalsRegistry,
 	}
 }
 
 // ApplyTo applies the run options to the method receiver and returns self
 func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
+	if err := s.ComponentGlobalsRegistry.SetFallback(); err != nil {
+		return err
+	}
 	c.CorsAllowedOriginList = s.CorsAllowedOriginList
 	c.HSTSDirectives = s.HSTSDirectives
 	c.ExternalAddress = s.ExternalHost
@@ -118,12 +142,15 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.RequestTimeout = s.RequestTimeout
 	c.GoawayChance = s.GoawayChance
 	c.MinRequestTimeout = s.MinRequestTimeout
+	c.StorageInitializationTimeout = s.StorageInitializationTimeout
 	c.ShutdownDelayDuration = s.ShutdownDelayDuration
 	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
 	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
 	c.PublicAddress = s.AdvertiseAddress
 	c.ShutdownSendRetryAfter = s.ShutdownSendRetryAfter
 	c.ShutdownWatchTerminationGracePeriod = s.ShutdownWatchTerminationGracePeriod
+	c.EffectiveVersion = s.ComponentGlobalsRegistry.EffectiveVersionFor(s.ComponentName)
+	c.FeatureGate = s.ComponentGlobalsRegistry.FeatureGateFor(s.ComponentName)
 
 	return nil
 }
@@ -173,6 +200,10 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
 	}
 
+	if s.StorageInitializationTimeout < 0 {
+		errors = append(errors, fmt.Errorf("--storage-initialization-timeout can not be negative value"))
+	}
+
 	if s.ShutdownDelayDuration < 0 {
 		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
 	}
@@ -196,6 +227,9 @@ func (s *ServerRunOptions) Validate() []error {
 	if err := validateCorsAllowedOriginList(s.CorsAllowedOriginList); err != nil {
 		errors = append(errors, err)
 	}
+	if errs := s.ComponentGlobalsRegistry.Validate(); len(errs) != 0 {
+		errors = append(errors, errs...)
+	}
 	return errors
 }
 
@@ -323,6 +357,9 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
+	fs.DurationVar(&s.StorageInitializationTimeout, "storage-initialization-timeout", s.StorageInitializationTimeout,
+		"Maximum amount of time to wait for storage initialization before declaring apiserver ready. Defaults to 1m.")
+
 	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
 		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
 		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
@@ -337,5 +374,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"This option, if set, represents the maximum amount of grace period the apiserver will wait "+
 		"for active watch request(s) to drain during the graceful server shutdown window.")
 
-	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
+	s.ComponentGlobalsRegistry.AddFlags(fs)
+}
+
+// Complete fills missing fields with defaults.
+func (s *ServerRunOptions) Complete() error {
+	return s.ComponentGlobalsRegistry.SetFallback()
 }

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -44,6 +44,8 @@ type SecureServingOptions struct {
 	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
 	// "tcp4", and "tcp6".
 	BindNetwork string
+	// DisableHTTP2Serving indicates that http2 serving should not be enabled.
+	DisableHTTP2Serving bool
 	// Required set to true means that BindPort cannot be zero.
 	Required bool
 	// ExternalAddress is the address advertised, even if BindAddress is a loopback. By default this
@@ -163,6 +165,9 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 	fs.IntVar(&s.BindPort, "secure-port", s.BindPort, desc)
 
+	fs.BoolVar(&s.DisableHTTP2Serving, "disable-http2-serving", s.DisableHTTP2Serving,
+		"If true, HTTP2 serving will be disabled [default=false]")
+
 	fs.StringVar(&s.ServerCert.CertDirectory, "cert-dir", s.ServerCert.CertDirectory, ""+
 		"The directory where the TLS certs are located. "+
 		"If --tls-cert-file and --tls-private-key-file are provided, this flag will be ignored.")
@@ -256,6 +261,7 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	*config = &server.SecureServingInfo{
 		Listener:                     s.Listener,
 		HTTP2MaxStreamsPerConnection: s.HTTP2MaxStreamsPerConnection,
+		DisableHTTP2:                 s.DisableHTTP2Serving,
 	}
 	c := *config
 

vendor/k8s.io/apiserver/pkg/server/routine/routine.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package filters
+package routine
 
 import (
 	"context"
@@ -35,6 +35,20 @@ func WithTask(parent context.Context, t *Task) context.Context {
 	return request.WithValue(parent, taskKey, t)
 }
 
+// AppendTask appends a task executed after completion of existing task.
+// It is a no-op if there is no existing task.
+func AppendTask(ctx context.Context, t *Task) bool {
+	if existTask := TaskFrom(ctx); existTask != nil && existTask.Func != nil {
+		existFunc := existTask.Func
+		existTask.Func = func() {
+			existFunc()
+			t.Func()
+		}
+		return true
+	}
+	return false
+}
+
 func TaskFrom(ctx context.Context) *Task {
 	t, _ := ctx.Value(taskKey).(*Task)
 	return t

vendor/k8s.io/apiserver/pkg/server/storage/resource_encoding_config.go

@@ -21,6 +21,8 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	apimachineryversion "k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apiserver/pkg/util/version"
 )
 
 type ResourceEncodingConfig interface {
@@ -33,10 +35,15 @@ type ResourceEncodingConfig interface {
 	InMemoryEncodingFor(schema.GroupResource) (schema.GroupVersion, error)
 }
 
+type CompatibilityResourceEncodingConfig interface {
+	BackwardCompatibileStorageEncodingFor(schema.GroupResource, runtime.Object) (schema.GroupVersion, error)
+}
+
 type DefaultResourceEncodingConfig struct {
 	// resources records the overriding encoding configs for individual resources.
-	resources map[schema.GroupResource]*OverridingResourceEncoding
-	scheme    *runtime.Scheme
+	resources        map[schema.GroupResource]*OverridingResourceEncoding
+	scheme           *runtime.Scheme
+	effectiveVersion version.EffectiveVersion
 }
 
 type OverridingResourceEncoding struct {
@@ -47,7 +54,7 @@ type OverridingResourceEncoding struct {
 var _ ResourceEncodingConfig = &DefaultResourceEncodingConfig{}
 
 func NewDefaultResourceEncodingConfig(scheme *runtime.Scheme) *DefaultResourceEncodingConfig {
-	return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme}
+	return &DefaultResourceEncodingConfig{resources: map[schema.GroupResource]*OverridingResourceEncoding{}, scheme: scheme, effectiveVersion: version.DefaultKubeEffectiveVersion()}
 }
 
 func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored schema.GroupResource, externalEncodingVersion, internalVersion schema.GroupVersion) {
@@ -57,6 +64,10 @@ func (o *DefaultResourceEncodingConfig) SetResourceEncoding(resourceBeingStored
 	}
 }
 
+func (o *DefaultResourceEncodingConfig) SetEffectiveVersion(effectiveVersion version.EffectiveVersion) {
+	o.effectiveVersion = effectiveVersion
+}
+
 func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
 	if !o.scheme.IsGroupRegistered(resource.Group) {
 		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -71,6 +82,24 @@ func (o *DefaultResourceEncodingConfig) StorageEncodingFor(resource schema.Group
 	return o.scheme.PrioritizedVersionsForGroup(resource.Group)[0], nil
 }
 
+func (o *DefaultResourceEncodingConfig) BackwardCompatibileStorageEncodingFor(resource schema.GroupResource, example runtime.Object) (schema.GroupVersion, error) {
+	if !o.scheme.IsGroupRegistered(resource.Group) {
+		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
+	}
+
+	// Always respect overrides
+	resourceOverride, resourceExists := o.resources[resource]
+	if resourceExists {
+		return resourceOverride.ExternalResourceEncoding, nil
+	}
+
+	return emulatedStorageVersion(
+		o.scheme.PrioritizedVersionsForGroup(resource.Group)[0],
+		example,
+		o.effectiveVersion,
+		o.scheme)
+}
+
 func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.GroupResource) (schema.GroupVersion, error) {
 	if !o.scheme.IsGroupRegistered(resource.Group) {
 		return schema.GroupVersion{}, fmt.Errorf("group %q is not registered in scheme", resource.Group)
@@ -82,3 +111,79 @@ func (o *DefaultResourceEncodingConfig) InMemoryEncodingFor(resource schema.Grou
 	}
 	return schema.GroupVersion{Group: resource.Group, Version: runtime.APIVersionInternal}, nil
 }
+
+// Object interface generated from "k8s:prerelease-lifecycle-gen:introduced" tags in types.go.
+type introducedInterface interface {
+	APILifecycleIntroduced() (major, minor int)
+}
+
+func emulatedStorageVersion(binaryVersionOfResource schema.GroupVersion, example runtime.Object, effectiveVersion version.EffectiveVersion, scheme *runtime.Scheme) (schema.GroupVersion, error) {
+	if example == nil || effectiveVersion == nil {
+		return binaryVersionOfResource, nil
+	}
+
+	// Look up example in scheme to find all objects of the same Group-Kind
+	// Use the highest priority version for that group-kind whose lifecycle window
+	// includes the current emulation version.
+	// If no version is found, use the binary version
+	// (in this case the API should be disabled anyway)
+	gvks, _, err := scheme.ObjectKinds(example)
+	if err != nil {
+		return schema.GroupVersion{}, err
+	} else if len(gvks) == 0 {
+		// Probably shouldn't happen if err is non-nil
+		return schema.GroupVersion{}, fmt.Errorf("object %T has no GVKs registered in scheme", example)
+	}
+
+	// VersionsForGroupKind returns versions in priority order
+	versions := scheme.VersionsForGroupKind(schema.GroupKind{Group: gvks[0].Group, Kind: gvks[0].Kind})
+
+	compatibilityVersion := effectiveVersion.MinCompatibilityVersion()
+
+	for _, gv := range versions {
+		if gv.Version == runtime.APIVersionInternal {
+			continue
+		}
+
+		gvk := schema.GroupVersionKind{
+			Group:   gv.Group,
+			Version: gv.Version,
+			Kind:    gvks[0].Kind,
+		}
+
+		exampleOfGVK, err := scheme.New(gvk)
+		if err != nil {
+			return schema.GroupVersion{}, err
+		}
+
+		// If it was introduced after current compatibility version, don't use it
+		// skip the introduced check for test when currentVersion is 0.0 to test all apis
+		if introduced, hasIntroduced := exampleOfGVK.(introducedInterface); hasIntroduced && (compatibilityVersion.Major() > 0 || compatibilityVersion.Minor() > 0) {
+			// API resource lifecycles should be relative to k8s api version
+			majorIntroduced, minorIntroduced := introduced.APILifecycleIntroduced()
+			introducedVer := apimachineryversion.MajorMinor(uint(majorIntroduced), uint(minorIntroduced))
+			if introducedVer.GreaterThan(compatibilityVersion) {
+				continue
+			}
+		}
+
+		// versions is returned in priority order, so just use first result
+		return gvk.GroupVersion(), nil
+	}
+
+	// Getting here means we're serving a version that is unknown to the
+	// min-compatibility-version server.
+	//
+	// This is only expected to happen when serving an alpha API type due
+	// to missing pre-release lifecycle information
+	// (which doesn't happen by default), or when emulation-version and
+	// min-compatibility-version are several versions apart so a beta or GA API
+	// was being served which didn't exist at all in min-compatibility-version.
+	//
+	// In the alpha case - we do not support compatibility versioning of
+	// alpha types and recommend users do not mix the two.
+	// In the skip-level case - The version of apiserver we are retaining
+	// compatibility with has no knowledge of the type,
+	// so storing it in another type is no issue.
+	return binaryVersionOfResource, nil
+}

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -42,7 +42,7 @@ type Backend struct {
 type StorageFactory interface {
 	// New finds the storage destination for the given group and resource. It will
 	// return an error if the group has no storage destination configured.
-	NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error)
+	NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error)
 
 	// ResourcePrefix returns the overridden resource prefix for the GroupResource
 	// This allows for cohabitation of resources with different native types and provides
@@ -226,7 +226,7 @@ func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.Gro
 
 // New finds the storage destination for the given group and resource. It will
 // return an error if the group has no storage destination configured.
-func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.ConfigForResource, error) {
+func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource, example runtime.Object) (*storagebackend.ConfigForResource, error) {
 	chosenStorageResource := s.getStorageGroupResource(groupResource)
 
 	// operate on copy
@@ -244,14 +244,23 @@ func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*
 	}
 
 	var err error
-	codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
-	if err != nil {
-		return nil, err
+	if backwardCompatibleInterface, ok := s.ResourceEncodingConfig.(CompatibilityResourceEncodingConfig); ok {
+		codecConfig.StorageVersion, err = backwardCompatibleInterface.BackwardCompatibileStorageEncodingFor(groupResource, example)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource)
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource)
 	if err != nil {
 		return nil, err
 	}
+
 	codecConfig.Config = storageConfig
 
 	storageConfig.Codec, storageConfig.EncodeVersioner, err = s.newStorageCodecFn(codecConfig)

vendor/k8s.io/apiserver/pkg/server/storage_readiness_hook.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2024 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package server
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/klog/v2"
+)
+
+// StorageReadinessHook implements PostStartHook functionality for checking readiness
+// of underlying storage for registered resources.
+type StorageReadinessHook struct {
+	timeout time.Duration
+
+	lock   sync.Mutex
+	checks map[string]func() error
+}
+
+// NewStorageReadinessHook created new StorageReadinessHook.
+func NewStorageReadinessHook(timeout time.Duration) *StorageReadinessHook {
+	return &StorageReadinessHook{
+		checks:  make(map[string]func() error),
+		timeout: timeout,
+	}
+}
+
+func (h *StorageReadinessHook) RegisterStorage(gvr metav1.GroupVersionResource, storage rest.StorageWithReadiness) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	if _, ok := h.checks[gvr.String()]; !ok {
+		h.checks[gvr.String()] = storage.ReadinessCheck
+	} else {
+		klog.Errorf("Registering storage readiness hook for %s again: ", gvr.String())
+	}
+}
+
+func (h *StorageReadinessHook) check() bool {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	failedChecks := []string{}
+	for gvr, check := range h.checks {
+		if err := check(); err != nil {
+			failedChecks = append(failedChecks, gvr)
+		}
+	}
+	if len(failedChecks) == 0 {
+		klog.Infof("Storage is ready for all registered resources")
+		return true
+	}
+	klog.V(4).Infof("Storage is not ready for: %v", failedChecks)
+	return false
+}
+
+func (h *StorageReadinessHook) Hook(ctx PostStartHookContext) error {
+	deadlineCtx, cancel := context.WithTimeout(ctx, h.timeout)
+	defer cancel()
+	err := wait.PollUntilContextCancel(deadlineCtx, 100*time.Millisecond, true,
+		func(_ context.Context) (bool, error) {
+			if ok := h.check(); ok {
+				return true, nil
+			}
+			return false, nil
+		})
+	if errors.Is(err, context.DeadlineExceeded) {
+		klog.Warningf("Deadline exceeded while waiting for storage readiness... ignoring")
+	}
+	return nil
+}