rebase: update kubernetes to 1.28.0 in main

updating kubernetes to 1.28.0
in the main repo.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

vendor/k8s.io/apiserver/pkg/admission/configuration/mutating_webhook_manager.go

@@ -19,8 +19,9 @@ package configuration
 import (
 	"fmt"
 	"sort"
+	"sync"
 
-	"k8s.io/api/admissionregistration/v1"
+	v1 "k8s.io/api/admissionregistration/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook"
@@ -29,13 +30,22 @@ import (
 	admissionregistrationlisters "k8s.io/client-go/listers/admissionregistration/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/cache/synctrack"
+	"k8s.io/klog/v2"
 )
 
+// Type for test injection.
+type mutatingWebhookAccessorCreator func(uid string, configurationName string, h *v1.MutatingWebhook) webhook.WebhookAccessor
+
 // mutatingWebhookConfigurationManager collects the mutating webhook objects so that they can be called.
 type mutatingWebhookConfigurationManager struct {
-	lister    admissionregistrationlisters.MutatingWebhookConfigurationLister
-	hasSynced func() bool
-	lazy      synctrack.Lazy[[]webhook.WebhookAccessor]
+	lister              admissionregistrationlisters.MutatingWebhookConfigurationLister
+	hasSynced           func() bool
+	lazy                synctrack.Lazy[[]webhook.WebhookAccessor]
+	configurationsCache sync.Map
+	// createMutatingWebhookAccessor is used to instantiate webhook accessors.
+	// This function is defined as field instead of a struct method to allow injection
+	// during tests
+	createMutatingWebhookAccessor mutatingWebhookAccessorCreator
 }
 
 var _ generic.Source = &mutatingWebhookConfigurationManager{}
@@ -43,14 +53,35 @@ var _ generic.Source = &mutatingWebhookConfigurationManager{}
 func NewMutatingWebhookConfigurationManager(f informers.SharedInformerFactory) generic.Source {
 	informer := f.Admissionregistration().V1().MutatingWebhookConfigurations()
 	manager := &mutatingWebhookConfigurationManager{
-		lister: informer.Lister(),
+		lister:                        informer.Lister(),
+		createMutatingWebhookAccessor: webhook.NewMutatingWebhookAccessor,
 	}
 	manager.lazy.Evaluate = manager.getConfiguration
 
 	handle, _ := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc:    func(_ interface{}) { manager.lazy.Notify() },
-		UpdateFunc: func(_, _ interface{}) { manager.lazy.Notify() },
-		DeleteFunc: func(_ interface{}) { manager.lazy.Notify() },
+		AddFunc: func(_ interface{}) { manager.lazy.Notify() },
+		UpdateFunc: func(old, new interface{}) {
+			obj := new.(*v1.MutatingWebhookConfiguration)
+			manager.configurationsCache.Delete(obj.GetName())
+			manager.lazy.Notify()
+		},
+		DeleteFunc: func(obj interface{}) {
+			vwc, ok := obj.(*v1.MutatingWebhookConfiguration)
+			if !ok {
+				tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+				if !ok {
+					klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
+					return
+				}
+				vwc, ok = tombstone.Obj.(*v1.MutatingWebhookConfiguration)
+				if !ok {
+					klog.V(2).Infof("Tombstone contained object that is not expected %#v", obj)
+					return
+				}
+			}
+			manager.configurationsCache.Delete(vwc.Name)
+			manager.lazy.Notify()
+		},
 	})
 	manager.hasSynced = handle.HasSynced
 
@@ -75,25 +106,46 @@ func (m *mutatingWebhookConfigurationManager) getConfiguration() ([]webhook.Webh
 	if err != nil {
 		return []webhook.WebhookAccessor{}, err
 	}
-	return mergeMutatingWebhookConfigurations(configurations), nil
+	return m.getMutatingWebhookConfigurations(configurations), nil
 }
 
-func mergeMutatingWebhookConfigurations(configurations []*v1.MutatingWebhookConfiguration) []webhook.WebhookAccessor {
+// getMutatingWebhookConfigurations returns the webhook accessors for a given list of
+// mutating webhook configurations.
+//
+// This function will, first, try to load the webhook accessors from the cache and avoid
+// recreating them, which can be expessive (requiring CEL expression recompilation).
+func (m *mutatingWebhookConfigurationManager) getMutatingWebhookConfigurations(configurations []*v1.MutatingWebhookConfiguration) []webhook.WebhookAccessor {
 	// The internal order of webhooks for each configuration is provided by the user
 	// but configurations themselves can be in any order. As we are going to run these
 	// webhooks in serial, they are sorted here to have a deterministic order.
 	sort.SliceStable(configurations, MutatingWebhookConfigurationSorter(configurations).ByName)
-	accessors := []webhook.WebhookAccessor{}
+	size := 0
+	for _, cfg := range configurations {
+		size += len(cfg.Webhooks)
+	}
+	accessors := make([]webhook.WebhookAccessor, 0, size)
+
 	for _, c := range configurations {
+		cachedConfigurationAccessors, ok := m.configurationsCache.Load(c.Name)
+		if ok {
+			// Pick an already cached webhookAccessor
+			accessors = append(accessors, cachedConfigurationAccessors.([]webhook.WebhookAccessor)...)
+			continue
+		}
+
 		// webhook names are not validated for uniqueness, so we check for duplicates and
 		// add a int suffix to distinguish between them
 		names := map[string]int{}
+		configurationAccessors := make([]webhook.WebhookAccessor, 0, len(c.Webhooks))
 		for i := range c.Webhooks {
 			n := c.Webhooks[i].Name
 			uid := fmt.Sprintf("%s/%s/%d", c.Name, n, names[n])
 			names[n]++
-			accessors = append(accessors, webhook.NewMutatingWebhookAccessor(uid, c.Name, &c.Webhooks[i]))
+			configurationAccessor := m.createMutatingWebhookAccessor(uid, c.Name, &c.Webhooks[i])
+			configurationAccessors = append(configurationAccessors, configurationAccessor)
 		}
+		accessors = append(accessors, configurationAccessors...)
+		m.configurationsCache.Store(c.Name, configurationAccessors)
 	}
 	return accessors
 }

vendor/k8s.io/apiserver/pkg/admission/configuration/validating_webhook_manager.go

@@ -19,8 +19,9 @@ package configuration
 import (
 	"fmt"
 	"sort"
+	"sync"
 
-	"k8s.io/api/admissionregistration/v1"
+	v1 "k8s.io/api/admissionregistration/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook"
@@ -29,13 +30,22 @@ import (
 	admissionregistrationlisters "k8s.io/client-go/listers/admissionregistration/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/cache/synctrack"
+	"k8s.io/klog/v2"
 )
 
+// Type for test injection.
+type validatingWebhookAccessorCreator func(uid string, configurationName string, h *v1.ValidatingWebhook) webhook.WebhookAccessor
+
 // validatingWebhookConfigurationManager collects the validating webhook objects so that they can be called.
 type validatingWebhookConfigurationManager struct {
-	lister    admissionregistrationlisters.ValidatingWebhookConfigurationLister
-	hasSynced func() bool
-	lazy      synctrack.Lazy[[]webhook.WebhookAccessor]
+	lister              admissionregistrationlisters.ValidatingWebhookConfigurationLister
+	hasSynced           func() bool
+	lazy                synctrack.Lazy[[]webhook.WebhookAccessor]
+	configurationsCache sync.Map
+	// createValidatingWebhookAccessor is used to instantiate webhook accessors.
+	// This function is defined as field instead of a struct method to allow injection
+	// during tests
+	createValidatingWebhookAccessor validatingWebhookAccessorCreator
 }
 
 var _ generic.Source = &validatingWebhookConfigurationManager{}
@@ -43,14 +53,35 @@ var _ generic.Source = &validatingWebhookConfigurationManager{}
 func NewValidatingWebhookConfigurationManager(f informers.SharedInformerFactory) generic.Source {
 	informer := f.Admissionregistration().V1().ValidatingWebhookConfigurations()
 	manager := &validatingWebhookConfigurationManager{
-		lister: informer.Lister(),
+		lister:                          informer.Lister(),
+		createValidatingWebhookAccessor: webhook.NewValidatingWebhookAccessor,
 	}
 	manager.lazy.Evaluate = manager.getConfiguration
 
 	handle, _ := informer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc:    func(_ interface{}) { manager.lazy.Notify() },
-		UpdateFunc: func(_, _ interface{}) { manager.lazy.Notify() },
-		DeleteFunc: func(_ interface{}) { manager.lazy.Notify() },
+		AddFunc: func(_ interface{}) { manager.lazy.Notify() },
+		UpdateFunc: func(old, new interface{}) {
+			obj := new.(*v1.ValidatingWebhookConfiguration)
+			manager.configurationsCache.Delete(obj.GetName())
+			manager.lazy.Notify()
+		},
+		DeleteFunc: func(obj interface{}) {
+			vwc, ok := obj.(*v1.ValidatingWebhookConfiguration)
+			if !ok {
+				tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
+				if !ok {
+					klog.V(2).Infof("Couldn't get object from tombstone %#v", obj)
+					return
+				}
+				vwc, ok = tombstone.Obj.(*v1.ValidatingWebhookConfiguration)
+				if !ok {
+					klog.V(2).Infof("Tombstone contained object that is not expected %#v", obj)
+					return
+				}
+			}
+			manager.configurationsCache.Delete(vwc.Name)
+			manager.lazy.Notify()
+		},
 	})
 	manager.hasSynced = handle.HasSynced
 
@@ -66,7 +97,7 @@ func (v *validatingWebhookConfigurationManager) Webhooks() []webhook.WebhookAcce
 	return out
 }
 
-// HasSynced returns true if the initial set of mutating webhook configurations
+// HasSynced returns true if the initial set of validating webhook configurations
 // has been loaded.
 func (v *validatingWebhookConfigurationManager) HasSynced() bool { return v.hasSynced() }
 
@@ -75,23 +106,45 @@ func (v *validatingWebhookConfigurationManager) getConfiguration() ([]webhook.We
 	if err != nil {
 		return []webhook.WebhookAccessor{}, err
 	}
-	return mergeValidatingWebhookConfigurations(configurations), nil
+	return v.getValidatingWebhookConfigurations(configurations), nil
 }
 
-func mergeValidatingWebhookConfigurations(configurations []*v1.ValidatingWebhookConfiguration) []webhook.WebhookAccessor {
+// getMutatingWebhookConfigurations returns the webhook accessors for a given list of
+// mutating webhook configurations.
+//
+// This function will, first, try to load the webhook accessors from the cache and avoid
+// recreating them, which can be expessive (requiring CEL expression recompilation).
+func (v *validatingWebhookConfigurationManager) getValidatingWebhookConfigurations(configurations []*v1.ValidatingWebhookConfiguration) []webhook.WebhookAccessor {
 	sort.SliceStable(configurations, ValidatingWebhookConfigurationSorter(configurations).ByName)
-	accessors := []webhook.WebhookAccessor{}
+	size := 0
+	for _, cfg := range configurations {
+		size += len(cfg.Webhooks)
+	}
+	accessors := make([]webhook.WebhookAccessor, 0, size)
+
 	for _, c := range configurations {
+		cachedConfigurationAccessors, ok := v.configurationsCache.Load(c.Name)
+		if ok {
+			// Pick an already cached webhookAccessor
+			accessors = append(accessors, cachedConfigurationAccessors.([]webhook.WebhookAccessor)...)
+			continue
+		}
+
 		// webhook names are not validated for uniqueness, so we check for duplicates and
 		// add a int suffix to distinguish between them
 		names := map[string]int{}
+		configurationAccessors := make([]webhook.WebhookAccessor, 0, len(c.Webhooks))
 		for i := range c.Webhooks {
 			n := c.Webhooks[i].Name
 			uid := fmt.Sprintf("%s/%s/%d", c.Name, n, names[n])
 			names[n]++
-			accessors = append(accessors, webhook.NewValidatingWebhookAccessor(uid, c.Name, &c.Webhooks[i]))
+			configurationAccessor := v.createValidatingWebhookAccessor(uid, c.Name, &c.Webhooks[i])
+			configurationAccessors = append(configurationAccessors, configurationAccessor)
 		}
+		accessors = append(accessors, configurationAccessors...)
+		v.configurationsCache.Store(c.Name, configurationAccessors)
 	}
+
 	return accessors
 }
 

vendor/k8s.io/apiserver/pkg/admission/metrics/metrics.go

@@ -54,6 +54,8 @@ var (
 type ObserverFunc func(ctx context.Context, elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string)
 
 const (
+	kindWebhook  = "webhook"
+	kindPolicy   = "policy"
 	stepValidate = "validate"
 	stepAdmit    = "admit"
 )
@@ -112,13 +114,15 @@ func (p pluginHandlerWithMetrics) Validate(ctx context.Context, a admission.Attr
 
 // AdmissionMetrics instruments admission with prometheus metrics.
 type AdmissionMetrics struct {
-	step                     *metricSet
-	controller               *metricSet
-	webhook                  *metricSet
-	webhookRejection         *metrics.CounterVec
-	webhookFailOpen          *metrics.CounterVec
-	webhookRequest           *metrics.CounterVec
-	matchConditionEvalErrors *metrics.CounterVec
+	step                            *metricSet
+	controller                      *metricSet
+	webhook                         *metricSet
+	webhookRejection                *metrics.CounterVec
+	webhookFailOpen                 *metrics.CounterVec
+	webhookRequest                  *metrics.CounterVec
+	matchConditionEvalErrors        *metrics.CounterVec
+	matchConditionExclusions        *metrics.CounterVec
+	matchConditionEvaluationSeconds *metricSet
 }
 
 // newAdmissionMetrics create a new AdmissionMetrics, configured with default metric names.
@@ -222,20 +226,47 @@ func newAdmissionMetrics() *AdmissionMetrics {
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
-			Name:           "admission_match_condition_evaluation_errors_total",
-			Help:           "Admission match condition evaluation errors count, identified by name of resource containing the match condition and broken out for each admission type (validating or mutating).",
+			Name:           "match_condition_evaluation_errors_total",
+			Help:           "Admission match condition evaluation errors count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"name", "type"})
+		[]string{"name", "kind", "type", "operation"})
+
+	matchConditionExclusions := metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "match_condition_exclusions_total",
+			Help:           "Admission match condition evaluation exclusions count, identified by name of resource containing the match condition and broken out for each kind containing matchConditions (webhook or policy), operation and admission type (validate or admit).",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"name", "kind", "type", "operation"})
+
+	matchConditionEvaluationSeconds := &metricSet{
+		latencies: metrics.NewHistogramVec(
+			&metrics.HistogramOpts{
+				Namespace:      namespace,
+				Subsystem:      subsystem,
+				Name:           "match_condition_evaluation_seconds",
+				Help:           "Admission match condition evaluation time in seconds, identified by name and broken out for each kind containing matchConditions (webhook or policy), operation and type (validate or admit).",
+				Buckets:        []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.2, 0.25},
+				StabilityLevel: metrics.ALPHA,
+			},
+			[]string{"name", "kind", "type", "operation"},
+		),
+		latenciesSummary: nil,
+	}
 
 	step.mustRegister()
 	controller.mustRegister()
 	webhook.mustRegister()
+	matchConditionEvaluationSeconds.mustRegister()
 	legacyregistry.MustRegister(webhookRejection)
 	legacyregistry.MustRegister(webhookFailOpen)
 	legacyregistry.MustRegister(webhookRequest)
 	legacyregistry.MustRegister(matchConditionEvalError)
-	return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest, matchConditionEvalErrors: matchConditionEvalError}
+	legacyregistry.MustRegister(matchConditionExclusions)
+	return &AdmissionMetrics{step: step, controller: controller, webhook: webhook, webhookRejection: webhookRejection, webhookFailOpen: webhookFailOpen, webhookRequest: webhookRequest, matchConditionEvalErrors: matchConditionEvalError, matchConditionExclusions: matchConditionExclusions, matchConditionEvaluationSeconds: matchConditionEvaluationSeconds}
 }
 
 func (m *AdmissionMetrics) reset() {
@@ -280,8 +311,18 @@ func (m *AdmissionMetrics) ObserveWebhookFailOpen(ctx context.Context, name, ste
 }
 
 // ObserveMatchConditionEvalError records validating or mutating webhook that are not called due to match conditions
-func (m *AdmissionMetrics) ObserveMatchConditionEvalError(ctx context.Context, name, stepType string) {
-	m.matchConditionEvalErrors.WithContext(ctx).WithLabelValues(name, stepType).Inc()
+func (m *AdmissionMetrics) ObserveMatchConditionEvalError(ctx context.Context, name, kind, stepType, operation string) {
+	m.matchConditionEvalErrors.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
+}
+
+// ObserveMatchConditionExclusion records validating or mutating webhook that are not called due to match conditions
+func (m *AdmissionMetrics) ObserveMatchConditionExclusion(ctx context.Context, name, kind, stepType, operation string) {
+	m.matchConditionExclusions.WithContext(ctx).WithLabelValues(name, kind, stepType, operation).Inc()
+}
+
+// ObserveMatchConditionEvaluationTime records duration of match condition evaluation process.
+func (m *AdmissionMetrics) ObserveMatchConditionEvaluationTime(ctx context.Context, elapsed time.Duration, name, kind, stepType, operation string) {
+	m.matchConditionEvaluationSeconds.observe(ctx, elapsed, name, kind, stepType, operation)
 }
 
 type metricSet struct {

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/compile.go

@@ -18,12 +18,13 @@ package cel
 
 import (
 	"fmt"
-	celconfig "k8s.io/apiserver/pkg/apis/cel"
-	"sync"
 
 	"github.com/google/cel-go/cel"
 
+	"k8s.io/apimachinery/pkg/util/version"
+	celconfig "k8s.io/apiserver/pkg/apis/cel"
 	apiservercel "k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
 	"k8s.io/apiserver/pkg/cel/library"
 )
 
@@ -32,108 +33,12 @@ const (
 	OldObjectVarName                 = "oldObject"
 	ParamsVarName                    = "params"
 	RequestVarName                   = "request"
+	NamespaceVarName                 = "namespaceObject"
 	AuthorizerVarName                = "authorizer"
 	RequestResourceAuthorizerVarName = "authorizer.requestResource"
+	VariableVarName                  = "variables"
 )
 
-var (
-	initEnvsOnce sync.Once
-	initEnvs     envs
-	initEnvsErr  error
-)
-
-func getEnvs() (envs, error) {
-	initEnvsOnce.Do(func() {
-		requiredVarsEnv, err := buildRequiredVarsEnv()
-		if err != nil {
-			initEnvsErr = err
-			return
-		}
-
-		initEnvs, err = buildWithOptionalVarsEnvs(requiredVarsEnv)
-		if err != nil {
-			initEnvsErr = err
-			return
-		}
-	})
-	return initEnvs, initEnvsErr
-}
-
-// This is a similar code as in k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel/compilation.go
-// If any changes are made here, consider to make the same changes there as well.
-func buildBaseEnv() (*cel.Env, error) {
-	var opts []cel.EnvOption
-	opts = append(opts, cel.HomogeneousAggregateLiterals())
-	// Validate function declarations once during base env initialization,
-	// so they don't need to be evaluated each time a CEL rule is compiled.
-	// This is a relatively expensive operation.
-	opts = append(opts, cel.EagerlyValidateDeclarations(true), cel.DefaultUTCTimeZone(true))
-	opts = append(opts, library.ExtensionLibs...)
-
-	return cel.NewEnv(opts...)
-}
-
-func buildRequiredVarsEnv() (*cel.Env, error) {
-	baseEnv, err := buildBaseEnv()
-	if err != nil {
-		return nil, err
-	}
-	var propDecls []cel.EnvOption
-	reg := apiservercel.NewRegistry(baseEnv)
-
-	requestType := BuildRequestType()
-	rt, err := apiservercel.NewRuleTypes(requestType.TypeName(), requestType, reg)
-	if err != nil {
-		return nil, err
-	}
-	if rt == nil {
-		return nil, nil
-	}
-	opts, err := rt.EnvOptions(baseEnv.TypeProvider())
-	if err != nil {
-		return nil, err
-	}
-	propDecls = append(propDecls, cel.Variable(ObjectVarName, cel.DynType))
-	propDecls = append(propDecls, cel.Variable(OldObjectVarName, cel.DynType))
-	propDecls = append(propDecls, cel.Variable(RequestVarName, requestType.CelType()))
-
-	opts = append(opts, propDecls...)
-	env, err := baseEnv.Extend(opts...)
-	if err != nil {
-		return nil, err
-	}
-	return env, nil
-}
-
-type envs map[OptionalVariableDeclarations]*cel.Env
-
-func buildEnvWithVars(baseVarsEnv *cel.Env, options OptionalVariableDeclarations) (*cel.Env, error) {
-	var opts []cel.EnvOption
-	if options.HasParams {
-		opts = append(opts, cel.Variable(ParamsVarName, cel.DynType))
-	}
-	if options.HasAuthorizer {
-		opts = append(opts, cel.Variable(AuthorizerVarName, library.AuthorizerType))
-		opts = append(opts, cel.Variable(RequestResourceAuthorizerVarName, library.ResourceCheckType))
-	}
-	return baseVarsEnv.Extend(opts...)
-}
-
-func buildWithOptionalVarsEnvs(requiredVarsEnv *cel.Env) (envs, error) {
-	envs := make(envs, 4) // since the number of variable combinations is small, pre-build a environment for each
-	for _, hasParams := range []bool{false, true} {
-		for _, hasAuthorizer := range []bool{false, true} {
-			opts := OptionalVariableDeclarations{HasParams: hasParams, HasAuthorizer: hasAuthorizer}
-			env, err := buildEnvWithVars(requiredVarsEnv, opts)
-			if err != nil {
-				return nil, err
-			}
-			envs[opts] = env
-		}
-	}
-	return envs, nil
-}
-
 // BuildRequestType generates a DeclType for AdmissionRequest. This may be replaced with a utility that
 // converts the native type definition to apiservercel.DeclType once such a utility becomes available.
 // The 'uid' field is omitted since it is not needed for in-process admission review.
@@ -181,6 +86,56 @@ func BuildRequestType() *apiservercel.DeclType {
 	))
 }
 
+// BuildNamespaceType generates a DeclType for Namespace.
+// Certain nested fields in Namespace (e.g. managedFields, ownerReferences etc.) are omitted in the generated DeclType
+// by design.
+func BuildNamespaceType() *apiservercel.DeclType {
+	field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
+		return apiservercel.NewDeclField(name, declType, required, nil, nil)
+	}
+	fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
+		result := make(map[string]*apiservercel.DeclField, len(fields))
+		for _, f := range fields {
+			result[f.Name] = f
+		}
+		return result
+	}
+
+	specType := apiservercel.NewObjectType("kubernetes.NamespaceSpec", fields(
+		field("finalizers", apiservercel.NewListType(apiservercel.StringType, -1), true),
+	))
+	conditionType := apiservercel.NewObjectType("kubernetes.NamespaceCondition", fields(
+		field("status", apiservercel.StringType, true),
+		field("type", apiservercel.StringType, true),
+		field("lastTransitionTime", apiservercel.TimestampType, true),
+		field("message", apiservercel.StringType, true),
+		field("reason", apiservercel.StringType, true),
+	))
+	statusType := apiservercel.NewObjectType("kubernetes.NamespaceStatus", fields(
+		field("conditions", apiservercel.NewListType(conditionType, -1), true),
+		field("phase", apiservercel.StringType, true),
+	))
+	metadataType := apiservercel.NewObjectType("kubernetes.NamespaceMetadata", fields(
+		field("name", apiservercel.StringType, true),
+		field("generateName", apiservercel.StringType, true),
+		field("namespace", apiservercel.StringType, true),
+		field("labels", apiservercel.NewMapType(apiservercel.StringType, apiservercel.StringType, -1), true),
+		field("annotations", apiservercel.NewMapType(apiservercel.StringType, apiservercel.StringType, -1), true),
+		field("UID", apiservercel.StringType, true),
+		field("creationTimestamp", apiservercel.TimestampType, true),
+		field("deletionGracePeriodSeconds", apiservercel.IntType, true),
+		field("deletionTimestamp", apiservercel.TimestampType, true),
+		field("generation", apiservercel.IntType, true),
+		field("resourceVersion", apiservercel.StringType, true),
+		field("finalizers", apiservercel.NewListType(apiservercel.StringType, -1), true),
+	))
+	return apiservercel.NewObjectType("kubernetes.Namespace", fields(
+		field("metadata", metadataType, true),
+		field("spec", specType, true),
+		field("status", statusType, true),
+	))
+}
+
 // CompilationResult represents a compiled validations expression.
 type CompilationResult struct {
 	Program            cel.Program
@@ -188,45 +143,48 @@ type CompilationResult struct {
 	ExpressionAccessor ExpressionAccessor
 }
 
+// Compiler provides a CEL expression compiler configured with the desired admission related CEL variables and
+// environment mode.
+type Compiler interface {
+	CompileCELExpression(expressionAccessor ExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) CompilationResult
+}
+
+type compiler struct {
+	varEnvs variableDeclEnvs
+}
+
+func NewCompiler(env *environment.EnvSet) Compiler {
+	return &compiler{varEnvs: mustBuildEnvs(env)}
+}
+
+type variableDeclEnvs map[OptionalVariableDeclarations]*environment.EnvSet
+
 // CompileCELExpression returns a compiled CEL expression.
 // perCallLimit was added for testing purpose only. Callers should always use const PerCallLimit from k8s.io/apiserver/pkg/apis/cel/config.go as input.
-func CompileCELExpression(expressionAccessor ExpressionAccessor, optionalVars OptionalVariableDeclarations, perCallLimit uint64) CompilationResult {
-	var env *cel.Env
-	envs, err := getEnvs()
-	if err != nil {
+func (c compiler) CompileCELExpression(expressionAccessor ExpressionAccessor, options OptionalVariableDeclarations, envType environment.Type) CompilationResult {
+	resultError := func(errorString string, errType apiservercel.ErrorType) CompilationResult {
 		return CompilationResult{
 			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInternal,
-				Detail: "compiler initialization failed: " + err.Error(),
-			},
-			ExpressionAccessor: expressionAccessor,
-		}
-	}
-	env, ok := envs[optionalVars]
-	if !ok {
-		return CompilationResult{
-			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInvalid,
-				Detail: fmt.Sprintf("compiler initialization failed: failed to load environment for %v", optionalVars),
+				Type:   errType,
+				Detail: errorString,
 			},
 			ExpressionAccessor: expressionAccessor,
 		}
 	}
 
+	env, err := c.varEnvs[options].Env(envType)
+	if err != nil {
+		return resultError(fmt.Sprintf("unexpected error loading CEL environment: %v", err), apiservercel.ErrorTypeInternal)
+	}
+
 	ast, issues := env.Compile(expressionAccessor.GetExpression())
 	if issues != nil {
-		return CompilationResult{
-			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInvalid,
-				Detail: "compilation failed: " + issues.String(),
-			},
-			ExpressionAccessor: expressionAccessor,
-		}
+		return resultError("compilation failed: "+issues.String(), apiservercel.ErrorTypeInvalid)
 	}
 	found := false
 	returnTypes := expressionAccessor.ReturnTypes()
 	for _, returnType := range returnTypes {
-		if ast.OutputType() == returnType {
+		if ast.OutputType() == returnType || cel.AnyType == returnType {
 			found = true
 			break
 		}
@@ -239,43 +197,64 @@ func CompileCELExpression(expressionAccessor ExpressionAccessor, optionalVars Op
 			reason = fmt.Sprintf("must evaluate to one of %v", returnTypes)
 		}
 
-		return CompilationResult{
-			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInvalid,
-				Detail: reason,
-			},
-			ExpressionAccessor: expressionAccessor,
-		}
+		return resultError(reason, apiservercel.ErrorTypeInvalid)
 	}
 
 	_, err = cel.AstToCheckedExpr(ast)
 	if err != nil {
 		// should be impossible since env.Compile returned no issues
-		return CompilationResult{
-			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInternal,
-				Detail: "unexpected compilation error: " + err.Error(),
-			},
-			ExpressionAccessor: expressionAccessor,
-		}
+		return resultError("unexpected compilation error: "+err.Error(), apiservercel.ErrorTypeInternal)
 	}
 	prog, err := env.Program(ast,
-		cel.EvalOptions(cel.OptOptimize, cel.OptTrackCost),
-		cel.OptimizeRegex(library.ExtensionLibRegexOptimizations...),
 		cel.InterruptCheckFrequency(celconfig.CheckFrequency),
-		cel.CostLimit(perCallLimit),
 	)
 	if err != nil {
-		return CompilationResult{
-			Error: &apiservercel.Error{
-				Type:   apiservercel.ErrorTypeInvalid,
-				Detail: "program instantiation failed: " + err.Error(),
-			},
-			ExpressionAccessor: expressionAccessor,
-		}
+		return resultError("program instantiation failed: "+err.Error(), apiservercel.ErrorTypeInternal)
 	}
 	return CompilationResult{
 		Program:            prog,
 		ExpressionAccessor: expressionAccessor,
 	}
 }
+
+func mustBuildEnvs(baseEnv *environment.EnvSet) variableDeclEnvs {
+	requestType := BuildRequestType()
+	namespaceType := BuildNamespaceType()
+	envs := make(variableDeclEnvs, 4) // since the number of variable combinations is small, pre-build a environment for each
+	for _, hasParams := range []bool{false, true} {
+		for _, hasAuthorizer := range []bool{false, true} {
+			var envOpts []cel.EnvOption
+			if hasParams {
+				envOpts = append(envOpts, cel.Variable(ParamsVarName, cel.DynType))
+			}
+			if hasAuthorizer {
+				envOpts = append(envOpts,
+					cel.Variable(AuthorizerVarName, library.AuthorizerType),
+					cel.Variable(RequestResourceAuthorizerVarName, library.ResourceCheckType))
+			}
+			envOpts = append(envOpts,
+				cel.Variable(ObjectVarName, cel.DynType),
+				cel.Variable(OldObjectVarName, cel.DynType),
+				cel.Variable(NamespaceVarName, namespaceType.CelType()),
+				cel.Variable(RequestVarName, requestType.CelType()))
+
+			extended, err := baseEnv.Extend(
+				environment.VersionedOptions{
+					// Feature epoch was actually 1.26, but we artificially set it to 1.0 because these
+					// options should always be present.
+					IntroducedVersion: version.MajorMinor(1, 0),
+					EnvOptions:        envOpts,
+					DeclTypes: []*apiservercel.DeclType{
+						namespaceType,
+						requestType,
+					},
+				},
+			)
+			if err != nil {
+				panic(fmt.Sprintf("environment misconfigured: %v", err))
+			}
+			envs[OptionalVariableDeclarations{HasParams: hasParams, HasAuthorizer: hasAuthorizer}] = extended
+		}
+	}
+	return envs
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/composition.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"math"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+
+	v1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apiserver/pkg/admission"
+	apiservercel "k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
+	"k8s.io/apiserver/pkg/cel/lazy"
+)
+
+const VariablesTypeName = "kubernetes.variables"
+
+type CompositedCompiler struct {
+	Compiler
+	FilterCompiler
+
+	CompositionEnv *CompositionEnv
+}
+
+type CompositedFilter struct {
+	Filter
+
+	compositionEnv *CompositionEnv
+}
+
+func NewCompositedCompiler(envSet *environment.EnvSet) (*CompositedCompiler, error) {
+	compositionContext, err := NewCompositionEnv(VariablesTypeName, envSet)
+	if err != nil {
+		return nil, err
+	}
+	compiler := NewCompiler(compositionContext.EnvSet)
+	filterCompiler := NewFilterCompiler(compositionContext.EnvSet)
+	return &CompositedCompiler{
+		Compiler:       compiler,
+		FilterCompiler: filterCompiler,
+		CompositionEnv: compositionContext,
+	}, nil
+}
+
+func (c *CompositedCompiler) CompileAndStoreVariables(variables []NamedExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) {
+	for _, v := range variables {
+		_ = c.CompileAndStoreVariable(v, options, mode)
+	}
+}
+
+func (c *CompositedCompiler) CompileAndStoreVariable(variable NamedExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) CompilationResult {
+	c.CompositionEnv.AddField(variable.GetName())
+	result := c.Compiler.CompileCELExpression(variable, options, mode)
+	c.CompositionEnv.CompiledVariables[variable.GetName()] = result
+	return result
+}
+
+func (c *CompositedCompiler) Compile(expressions []ExpressionAccessor, optionalDecls OptionalVariableDeclarations, envType environment.Type) Filter {
+	filter := c.FilterCompiler.Compile(expressions, optionalDecls, envType)
+	return &CompositedFilter{
+		Filter:         filter,
+		compositionEnv: c.CompositionEnv,
+	}
+}
+
+type CompositionEnv struct {
+	*environment.EnvSet
+
+	MapType           *apiservercel.DeclType
+	CompiledVariables map[string]CompilationResult
+}
+
+func (c *CompositionEnv) AddField(name string) {
+	c.MapType.Fields[name] = apiservercel.NewDeclField(name, apiservercel.DynType, true, nil, nil)
+}
+
+func NewCompositionEnv(typeName string, baseEnvSet *environment.EnvSet) (*CompositionEnv, error) {
+	declType := apiservercel.NewObjectType(typeName, map[string]*apiservercel.DeclField{})
+	envSet, err := baseEnvSet.Extend(environment.VersionedOptions{
+		// set to 1.0 because composition is one of the fundamental components
+		IntroducedVersion: version.MajorMinor(1, 0),
+		EnvOptions: []cel.EnvOption{
+			cel.Variable("variables", declType.CelType()),
+		},
+		DeclTypes: []*apiservercel.DeclType{
+			declType,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &CompositionEnv{
+		MapType:           declType,
+		EnvSet:            envSet,
+		CompiledVariables: map[string]CompilationResult{},
+	}, nil
+}
+
+func (c *CompositionEnv) CreateContext(parent context.Context) CompositionContext {
+	return &compositionContext{
+		Context:        parent,
+		compositionEnv: c,
+	}
+}
+
+type CompositionContext interface {
+	context.Context
+	Variables(activation any) ref.Val
+	GetAndResetCost() int64
+}
+
+type compositionContext struct {
+	context.Context
+
+	compositionEnv  *CompositionEnv
+	accumulatedCost int64
+}
+
+func (c *compositionContext) Variables(activation any) ref.Val {
+	lazyMap := lazy.NewMapValue(c.compositionEnv.MapType)
+	for name, result := range c.compositionEnv.CompiledVariables {
+		accessor := &variableAccessor{
+			name:       name,
+			result:     result,
+			activation: activation,
+			context:    c,
+		}
+		lazyMap.Append(name, accessor.Callback)
+	}
+	return lazyMap
+}
+
+func (f *CompositedFilter) ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *v1.AdmissionRequest, optionalVars OptionalVariableBindings, namespace *corev1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error) {
+	ctx = f.compositionEnv.CreateContext(ctx)
+	return f.Filter.ForInput(ctx, versionedAttr, request, optionalVars, namespace, runtimeCELCostBudget)
+}
+
+func (c *compositionContext) reportCost(cost int64) {
+	c.accumulatedCost += cost
+}
+
+func (c *compositionContext) GetAndResetCost() int64 {
+	cost := c.accumulatedCost
+	c.accumulatedCost = 0
+	return cost
+}
+
+type variableAccessor struct {
+	name       string
+	result     CompilationResult
+	activation any
+	context    *compositionContext
+}
+
+func (a *variableAccessor) Callback(_ *lazy.MapValue) ref.Val {
+	if a.result.Error != nil {
+		return types.NewErr("composited variable %q fails to compile: %v", a.name, a.result.Error)
+	}
+
+	v, details, err := a.result.Program.Eval(a.activation)
+	if details == nil {
+		return types.NewErr("unable to get evaluation details of variable %q", a.name)
+	}
+	costPtr := details.ActualCost()
+	if costPtr == nil {
+		return types.NewErr("unable to calculate cost of variable %q", a.name)
+	}
+	cost := int64(*costPtr)
+	if *costPtr > math.MaxInt64 {
+		cost = math.MaxInt64
+	}
+	a.context.reportCost(cost)
+
+	if err != nil {
+		return types.NewErr("composited variable %q fails to evaluate: %v", a.name, err)
+	}
+	return v
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/filter.go

@@ -27,24 +27,27 @@ import (
 
 	admissionv1 "k8s.io/api/admission/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
 	"k8s.io/apiserver/pkg/cel/library"
 )
 
 // filterCompiler implement the interface FilterCompiler.
 type filterCompiler struct {
+	compiler Compiler
 }
 
-func NewFilterCompiler() FilterCompiler {
-	return &filterCompiler{}
+func NewFilterCompiler(env *environment.EnvSet) FilterCompiler {
+	return &filterCompiler{compiler: NewCompiler(env)}
 }
 
 type evaluationActivation struct {
-	object, oldObject, params, request, authorizer, requestResourceAuthorizer interface{}
+	object, oldObject, params, request, namespace, authorizer, requestResourceAuthorizer, variables interface{}
 }
 
 // ResolveName returns a value from the activation by qualified name, or false if the name
@@ -59,10 +62,14 @@ func (a *evaluationActivation) ResolveName(name string) (interface{}, bool) {
 		return a.params, true // params may be null
 	case RequestVarName:
 		return a.request, true
+	case NamespaceVarName:
+		return a.namespace, true
 	case AuthorizerVarName:
 		return a.authorizer, a.authorizer != nil
 	case RequestResourceAuthorizerVarName:
 		return a.requestResourceAuthorizer, a.requestResourceAuthorizer != nil
+	case VariableVarName: // variables always present
+		return a.variables, true
 	default:
 		return nil, false
 	}
@@ -75,13 +82,13 @@ func (a *evaluationActivation) Parent() interpreter.Activation {
 }
 
 // Compile compiles the cel expressions defined in the ExpressionAccessors into a Filter
-func (c *filterCompiler) Compile(expressionAccessors []ExpressionAccessor, options OptionalVariableDeclarations, perCallLimit uint64) Filter {
+func (c *filterCompiler) Compile(expressionAccessors []ExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) Filter {
 	compilationResults := make([]CompilationResult, len(expressionAccessors))
 	for i, expressionAccessor := range expressionAccessors {
 		if expressionAccessor == nil {
 			continue
 		}
-		compilationResults[i] = CompileCELExpression(expressionAccessor, options, perCallLimit)
+		compilationResults[i] = c.compiler.CompileCELExpression(expressionAccessor, options, mode)
 	}
 	return NewFilter(compilationResults)
 }
@@ -122,7 +129,7 @@ func objectToResolveVal(r runtime.Object) (interface{}, error) {
 // ForInput evaluates the compiled CEL expressions converting them into CELEvaluations
 // errors per evaluation are returned on the Evaluation object
 // runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
-func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *admissionv1.AdmissionRequest, inputs OptionalVariableBindings, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error) {
+func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *admissionv1.AdmissionRequest, inputs OptionalVariableBindings, namespace *v1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error) {
 	// TODO: replace unstructured with ref.Val for CEL variables when native type support is available
 	evaluations := make([]EvaluationResult, len(f.compilationResults))
 	var err error
@@ -152,15 +159,28 @@ func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.Versione
 	if err != nil {
 		return nil, -1, err
 	}
+	namespaceVal, err := objectToResolveVal(namespace)
+	if err != nil {
+		return nil, -1, err
+	}
 	va := &evaluationActivation{
 		object:                    objectVal,
 		oldObject:                 oldObjectVal,
 		params:                    paramsVal,
 		request:                   requestVal.Object,
+		namespace:                 namespaceVal,
 		authorizer:                authorizerVal,
 		requestResourceAuthorizer: requestResourceAuthorizerVal,
 	}
 
+	// composition is an optional feature that only applies for ValidatingAdmissionPolicy.
+	// check if the context allows composition
+	var compositionCtx CompositionContext
+	var ok bool
+	if compositionCtx, ok = ctx.(CompositionContext); ok {
+		va.variables = compositionCtx.Variables(va)
+	}
+
 	remainingBudget := runtimeCELCostBudget
 	for i, compilationResult := range f.compilationResults {
 		var evaluation = &evaluations[i]
@@ -184,6 +204,17 @@ func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.Versione
 		}
 		t1 := time.Now()
 		evalResult, evalDetails, err := compilationResult.Program.ContextEval(ctx, va)
+		// budget may be spent due to lazy evaluation of composited variables
+		if compositionCtx != nil {
+			compositionCost := compositionCtx.GetAndResetCost()
+			if compositionCost > remainingBudget {
+				return nil, -1, &cel.Error{
+					Type:   cel.ErrorTypeInvalid,
+					Detail: fmt.Sprintf("validation failed due to running out of cost budget, no further validation rules will be run"),
+				}
+			}
+			remainingBudget -= compositionCost
+		}
 		elapsed := time.Since(t1)
 		evaluation.Elapsed = elapsed
 		if evalDetails == nil {
@@ -222,10 +253,13 @@ func (f *filter) ForInput(ctx context.Context, versionedAttr *admission.Versione
 }
 
 // TODO: to reuse https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/request/admissionreview.go#L154
-func CreateAdmissionRequest(attr admission.Attributes) *admissionv1.AdmissionRequest {
-	// FIXME: how to get resource GVK, GVR and subresource?
-	gvk := attr.GetKind()
-	gvr := attr.GetResource()
+func CreateAdmissionRequest(attr admission.Attributes, equivalentGVR metav1.GroupVersionResource, equivalentKind metav1.GroupVersionKind) *admissionv1.AdmissionRequest {
+	// Attempting to use same logic as webhook for constructing resource
+	// GVK, GVR, subresource
+	// Use the GVK, GVR that the matcher decided was equivalent to that of the request
+	// https://github.com/kubernetes/kubernetes/blob/90c362b3430bcbbf8f245fadbcd521dab39f1d7c/staging/src/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go#L182-L210
+	gvk := equivalentKind
+	gvr := equivalentGVR
 	subresource := attr.GetSubresource()
 
 	requestGVK := attr.GetKind()
@@ -284,6 +318,33 @@ func CreateAdmissionRequest(attr admission.Attributes) *admissionv1.AdmissionReq
 	}
 }
 
+// CreateNamespaceObject creates a Namespace object that is suitable for the CEL evaluation.
+// If the namespace is nil, CreateNamespaceObject returns nil
+func CreateNamespaceObject(namespace *v1.Namespace) *v1.Namespace {
+	if namespace == nil {
+		return nil
+	}
+
+	return &v1.Namespace{
+		Status: namespace.Status,
+		Spec:   namespace.Spec,
+		ObjectMeta: metav1.ObjectMeta{
+			Name:                       namespace.Name,
+			GenerateName:               namespace.GenerateName,
+			Namespace:                  namespace.Namespace,
+			UID:                        namespace.UID,
+			ResourceVersion:            namespace.ResourceVersion,
+			Generation:                 namespace.Generation,
+			CreationTimestamp:          namespace.CreationTimestamp,
+			DeletionTimestamp:          namespace.DeletionTimestamp,
+			DeletionGracePeriodSeconds: namespace.DeletionGracePeriodSeconds,
+			Labels:                     namespace.Labels,
+			Annotations:                namespace.Annotations,
+			Finalizers:                 namespace.Finalizers,
+		},
+	}
+}
+
 // CompilationErrors returns a list of all the errors from the compilation of the evaluator
 func (e *filter) CompilationErrors() []error {
 	compilationErrors := []error{}

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/interface.go

@@ -24,9 +24,11 @@ import (
 	"github.com/google/cel-go/common/types/ref"
 
 	v1 "k8s.io/api/admission/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/cel/environment"
 )
 
 type ExpressionAccessor interface {
@@ -34,6 +36,13 @@ type ExpressionAccessor interface {
 	ReturnTypes() []*cel.Type
 }
 
+// NamedExpressionAccessor extends NamedExpressionAccessor with a name.
+type NamedExpressionAccessor interface {
+	ExpressionAccessor
+
+	GetName() string // follows the naming convention of ExpressionAccessor
+}
+
 // EvaluationResult contains the minimal required fields and metadata of a cel evaluation
 type EvaluationResult struct {
 	EvalResult         ref.Val
@@ -57,8 +66,7 @@ type OptionalVariableDeclarations struct {
 // FilterCompiler contains a function to assist with converting types and values to/from CEL-typed values.
 type FilterCompiler interface {
 	// Compile is used for the cel expression compilation
-	// perCallLimit was added for testing purpose only. Callers should always use const PerCallLimit from k8s.io/apiserver/pkg/apis/cel/config.go as input.
-	Compile(expressions []ExpressionAccessor, optionalDecls OptionalVariableDeclarations, perCallLimit uint64) Filter
+	Compile(expressions []ExpressionAccessor, optionalDecls OptionalVariableDeclarations, envType environment.Type) Filter
 }
 
 // OptionalVariableBindings provides expression bindings for optional CEL variables.
@@ -80,7 +88,7 @@ type Filter interface {
 	// ForInput converts compiled CEL-typed values into evaluated CEL-typed value.
 	// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
 	// If cost budget is calculated, the filter should return the remaining budget.
-	ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *v1.AdmissionRequest, optionalVars OptionalVariableBindings, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error)
+	ForInput(ctx context.Context, versionedAttr *admission.VersionedAttributes, request *v1.AdmissionRequest, optionalVars OptionalVariableBindings, namespace *corev1.Namespace, runtimeCELCostBudget int64) ([]EvaluationResult, int64, error)
 
 	// CompilationErrors returns a list of errors from the compilation of the evaluator
 	CompilationErrors() []error

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/admission.go

@@ -24,7 +24,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/cel/openapi/resolver"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/component-base/featuregate"
@@ -74,7 +73,6 @@ type celAdmissionPlugin struct {
 	dynamicClient   dynamic.Interface
 	stopCh          <-chan struct{}
 	authorizer      authorizer.Authorizer
-	schemaResolver  resolver.SchemaResolver
 }
 
 var _ initializer.WantsExternalKubeInformerFactory = &celAdmissionPlugin{}
@@ -83,7 +81,6 @@ var _ initializer.WantsRESTMapper = &celAdmissionPlugin{}
 var _ initializer.WantsDynamicClient = &celAdmissionPlugin{}
 var _ initializer.WantsDrainedNotification = &celAdmissionPlugin{}
 var _ initializer.WantsAuthorizer = &celAdmissionPlugin{}
-var _ initializer.WantsSchemaResolver = &celAdmissionPlugin{}
 var _ admission.InitializationValidator = &celAdmissionPlugin{}
 var _ admission.ValidationInterface = &celAdmissionPlugin{}
 
@@ -116,11 +113,6 @@ func (c *celAdmissionPlugin) SetDrainedNotification(stopCh <-chan struct{}) {
 func (c *celAdmissionPlugin) SetAuthorizer(authorizer authorizer.Authorizer) {
 	c.authorizer = authorizer
 }
-
-func (c *celAdmissionPlugin) SetSchemaResolver(resolver resolver.SchemaResolver) {
-	c.schemaResolver = resolver
-}
-
 func (c *celAdmissionPlugin) InspectFeatureGates(featureGates featuregate.FeatureGate) {
 	if featureGates.Enabled(features.ValidatingAdmissionPolicy) {
 		c.enabled = true
@@ -154,7 +146,7 @@ func (c *celAdmissionPlugin) ValidateInitialization() error {
 	if c.authorizer == nil {
 		return errors.New("missing authorizer")
 	}
-	c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.schemaResolver /* (optional) */, c.dynamicClient, c.authorizer)
+	c.evaluator = NewAdmissionController(c.informerFactory, c.client, c.restMapper, c.dynamicClient, c.authorizer)
 	if err := c.evaluator.ValidateInitialization(); err != nil {
 		return err
 	}

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/caching_authorizer.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validatingadmissionpolicy
+
+import (
+	"context"
+	"encoding/json"
+	"sort"
+	"strings"
+
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+)
+
+type authzResult struct {
+	authorized authorizer.Decision
+	reason     string
+	err        error
+}
+
+type cachingAuthorizer struct {
+	authorizer authorizer.Authorizer
+	decisions  map[string]authzResult
+}
+
+func newCachingAuthorizer(in authorizer.Authorizer) authorizer.Authorizer {
+	return &cachingAuthorizer{
+		authorizer: in,
+		decisions:  make(map[string]authzResult),
+	}
+}
+
+// The attribute accessors known to cache key construction. If this fails to compile, the cache
+// implementation may need to be updated.
+var _ authorizer.Attributes = (interface {
+	GetUser() user.Info
+	GetVerb() string
+	IsReadOnly() bool
+	GetNamespace() string
+	GetResource() string
+	GetSubresource() string
+	GetName() string
+	GetAPIGroup() string
+	GetAPIVersion() string
+	IsResourceRequest() bool
+	GetPath() string
+})(nil)
+
+// The user info accessors known to cache key construction. If this fails to compile, the cache
+// implementation may need to be updated.
+var _ user.Info = (interface {
+	GetName() string
+	GetUID() string
+	GetGroups() []string
+	GetExtra() map[string][]string
+})(nil)
+
+// Authorize returns an authorization decision by delegating to another Authorizer. If an equivalent
+// check has already been performed, a cached result is returned. Not safe for concurrent use.
+func (ca *cachingAuthorizer) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
+	serializableAttributes := authorizer.AttributesRecord{
+		Verb:            a.GetVerb(),
+		Namespace:       a.GetNamespace(),
+		APIGroup:        a.GetAPIGroup(),
+		APIVersion:      a.GetAPIVersion(),
+		Resource:        a.GetResource(),
+		Subresource:     a.GetSubresource(),
+		Name:            a.GetName(),
+		ResourceRequest: a.IsResourceRequest(),
+		Path:            a.GetPath(),
+	}
+
+	if u := a.GetUser(); u != nil {
+		di := &user.DefaultInfo{
+			Name: u.GetName(),
+			UID:  u.GetUID(),
+		}
+
+		// Differently-ordered groups or extras could cause otherwise-equivalent checks to
+		// have distinct cache keys.
+		if groups := u.GetGroups(); len(groups) > 0 {
+			di.Groups = make([]string, len(groups))
+			copy(di.Groups, groups)
+			sort.Strings(di.Groups)
+		}
+
+		if extra := u.GetExtra(); len(extra) > 0 {
+			di.Extra = make(map[string][]string, len(extra))
+			for k, vs := range extra {
+				vdupe := make([]string, len(vs))
+				copy(vdupe, vs)
+				sort.Strings(vdupe)
+				di.Extra[k] = vdupe
+			}
+		}
+
+		serializableAttributes.User = di
+	}
+
+	var b strings.Builder
+	if err := json.NewEncoder(&b).Encode(serializableAttributes); err != nil {
+		return authorizer.DecisionNoOpinion, "", err
+	}
+	key := b.String()
+
+	if cached, ok := ca.decisions[key]; ok {
+		return cached.authorized, cached.reason, cached.err
+	}
+
+	authorized, reason, err := ca.authorizer.Authorize(ctx, a)
+
+	ca.decisions[key] = authzResult{
+		authorized: authorized,
+		reason:     reason,
+		err:        err,
+	}
+
+	return authorized, reason, err
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller.go

@@ -25,30 +25,29 @@ import (
 	"sync/atomic"
 	"time"
 
-	"k8s.io/klog/v2"
-
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
+	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	utiljson "k8s.io/apimachinery/pkg/util/json"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/admission"
 	celmetrics "k8s.io/apiserver/pkg/admission/cel"
-	"k8s.io/apiserver/pkg/admission/plugin/cel"
 	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
 	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching"
 	celconfig "k8s.io/apiserver/pkg/apis/cel"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
-	"k8s.io/apiserver/pkg/cel/openapi/resolver"
 	"k8s.io/apiserver/pkg/warning"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
 )
 
 var _ CELPolicyEvaluator = &celAdmissionController{}
@@ -66,22 +65,24 @@ type celAdmissionController struct {
 	// A snapshot of the current policy configuration is synced with this field
 	// asynchronously
 	definitions atomic.Value
+
+	authz authorizer.Authorizer
 }
 
 // Everything someone might need to validate a single ValidatingPolicyDefinition
 // against all of its registered bindings.
 type policyData struct {
 	definitionInfo
-	paramController generic.Controller[runtime.Object]
-	bindings        []bindingInfo
+	paramInfo
+	bindings []bindingInfo
 }
 
 // contains the cel PolicyDecisions along with the ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding
 // that determined the decision
 type policyDecisionWithMetadata struct {
 	PolicyDecision
-	Definition *v1alpha1.ValidatingAdmissionPolicy
-	Binding    *v1alpha1.ValidatingAdmissionPolicyBinding
+	Definition *v1beta1.ValidatingAdmissionPolicy
+	Binding    *v1beta1.ValidatingAdmissionPolicyBinding
 }
 
 // namespaceName is used as a key in definitionInfo and bindingInfos
@@ -97,7 +98,7 @@ type definitionInfo struct {
 
 	// Last value seen by this controller to be used in policy enforcement
 	// May not be nil
-	lastReconciledValue *v1alpha1.ValidatingAdmissionPolicy
+	lastReconciledValue *v1beta1.ValidatingAdmissionPolicy
 }
 
 type bindingInfo struct {
@@ -106,7 +107,7 @@ type bindingInfo struct {
 
 	// Last value seen by this controller to be used in policy enforcement
 	// May not be nil
-	lastReconciledValue *v1alpha1.ValidatingAdmissionPolicyBinding
+	lastReconciledValue *v1beta1.ValidatingAdmissionPolicyBinding
 }
 
 type paramInfo struct {
@@ -116,6 +117,9 @@ type paramInfo struct {
 	// Function to call to stop the informer and clean up the controller
 	stop func()
 
+	// Whether this param is cluster or namespace scoped
+	scope meta.RESTScope
+
 	// Policy Definitions which refer to this param CRD
 	dependentDefinitions sets.Set[namespacedName]
 }
@@ -125,29 +129,24 @@ func NewAdmissionController(
 	informerFactory informers.SharedInformerFactory,
 	client kubernetes.Interface,
 	restMapper meta.RESTMapper,
-	schemaResolver resolver.SchemaResolver,
 	dynamicClient dynamic.Interface,
 	authz authorizer.Authorizer,
 ) CELPolicyEvaluator {
-	var typeChecker *TypeChecker
-	if schemaResolver != nil {
-		typeChecker = &TypeChecker{schemaResolver: schemaResolver, restMapper: restMapper}
-	}
 	return &celAdmissionController{
 		definitions: atomic.Value{},
 		policyController: newPolicyController(
 			restMapper,
 			client,
 			dynamicClient,
-			typeChecker,
-			cel.NewFilterCompiler(),
+			informerFactory,
+			nil,
 			NewMatcher(matching.NewMatcher(informerFactory.Core().V1().Namespaces().Lister(), client)),
-			generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicy](
-				informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicies().Informer()),
-			generic.NewInformer[*v1alpha1.ValidatingAdmissionPolicyBinding](
-				informerFactory.Admissionregistration().V1alpha1().ValidatingAdmissionPolicyBindings().Informer()),
-			authz,
+			generic.NewInformer[*v1beta1.ValidatingAdmissionPolicy](
+				informerFactory.Admissionregistration().V1beta1().ValidatingAdmissionPolicies().Informer()),
+			generic.NewInformer[*v1beta1.ValidatingAdmissionPolicyBinding](
+				informerFactory.Admissionregistration().V1beta1().ValidatingAdmissionPolicyBindings().Informer()),
 		),
+		authz: authz,
 	}
 }
 
@@ -193,21 +192,21 @@ func (c *celAdmissionController) Validate(
 
 	var deniedDecisions []policyDecisionWithMetadata
 
-	addConfigError := func(err error, definition *v1alpha1.ValidatingAdmissionPolicy, binding *v1alpha1.ValidatingAdmissionPolicyBinding) {
+	addConfigError := func(err error, definition *v1beta1.ValidatingAdmissionPolicy, binding *v1beta1.ValidatingAdmissionPolicyBinding) {
 		// we always default the FailurePolicy if it is unset and validate it in API level
-		var policy v1alpha1.FailurePolicyType
+		var policy v1beta1.FailurePolicyType
 		if definition.Spec.FailurePolicy == nil {
-			policy = v1alpha1.Fail
+			policy = v1beta1.Fail
 		} else {
 			policy = *definition.Spec.FailurePolicy
 		}
 
 		// apply FailurePolicy specified in ValidatingAdmissionPolicy, the default would be Fail
 		switch policy {
-		case v1alpha1.Ignore:
+		case v1beta1.Ignore:
 			// TODO: add metrics for ignored error here
 			return
-		case v1alpha1.Fail:
+		case v1beta1.Fail:
 			var message string
 			if binding == nil {
 				message = fmt.Errorf("failed to configure policy: %w", err).Error()
@@ -235,9 +234,17 @@ func (c *celAdmissionController) Validate(
 	}
 	policyDatas := c.definitions.Load().([]policyData)
 
+	authz := newCachingAuthorizer(c.authz)
+
 	for _, definitionInfo := range policyDatas {
+		// versionedAttributes will be set to non-nil inside of the loop, but
+		// is scoped outside of the param loop so we only convert once. We defer
+		// conversion so that it is only performed when we know a policy matches,
+		// saving the cost of converting non-matching requests.
+		var versionedAttr *admission.VersionedAttributes
+
 		definition := definitionInfo.lastReconciledValue
-		matches, matchKind, err := c.policyController.matcher.DefinitionMatches(a, o, definition)
+		matches, matchResource, matchKind, err := c.policyController.matcher.DefinitionMatches(a, o, definition)
 		if err != nil {
 			// Configuration error.
 			addConfigError(err, definition, nil)
@@ -267,65 +274,13 @@ func (c *celAdmissionController) Validate(
 				continue
 			}
 
-			var param runtime.Object
-
-			// versionedAttributes will be set to non-nil inside of the loop, but
-			// is scoped outside of the param loop so we only convert once. We defer
-			// conversion so that it is only performed when we know a policy matches,
-			// saving the cost of converting non-matching requests.
-			var versionedAttr *admission.VersionedAttributes
-
-			// If definition has paramKind, paramRef is required in binding.
-			// If definition has no paramKind, paramRef set in binding will be ignored.
-			paramKind := definition.Spec.ParamKind
-			paramRef := binding.Spec.ParamRef
-			if paramKind != nil && paramRef != nil {
-				paramController := definitionInfo.paramController
-				if paramController == nil {
-					addConfigError(fmt.Errorf("paramKind kind `%v` not known",
-						paramKind.String()), definition, binding)
-					continue
-				}
-
-				// If the param informer for this admission policy has not yet
-				// had time to perform an initial listing, don't attempt to use
-				// it.
-				timeoutCtx, cancel := context.WithTimeout(c.policyController.context, 1*time.Second)
-				defer cancel()
-
-				if !cache.WaitForCacheSync(timeoutCtx.Done(), paramController.HasSynced) {
-					addConfigError(fmt.Errorf("paramKind kind `%v` not yet synced to use for admission",
-						paramKind.String()), definition, binding)
-					continue
-				}
-
-				if len(paramRef.Namespace) == 0 {
-					param, err = paramController.Informer().Get(paramRef.Name)
-				} else {
-					param, err = paramController.Informer().Namespaced(paramRef.Namespace).Get(paramRef.Name)
-				}
-
-				if err != nil {
-					// Apply failure policy
-					addConfigError(err, definition, binding)
-
-					if k8serrors.IsInvalid(err) {
-						// Param mis-configured
-						// require to set paramRef.namespace for namespaced resource and unset paramRef.namespace for cluster scoped resource
-						continue
-					} else if k8serrors.IsNotFound(err) {
-						// Param not yet available. User may need to wait a bit
-						// before being able to use it for validation.
-						continue
-					}
-
-					// There was a bad internal error
-					utilruntime.HandleError(err)
-					continue
-				}
-			}
-
-			if versionedAttr == nil {
+			params, err := c.collectParams(definition.Spec.ParamKind, definitionInfo.paramInfo, binding.Spec.ParamRef, a.GetNamespace())
+			if err != nil {
+				addConfigError(err, definition, binding)
+				continue
+			} else if versionedAttr == nil && len(params) > 0 {
+				// As optimization versionedAttr creation is deferred until
+				// first use. Since > 0 params, we will validate
 				va, err := admission.NewVersionedAttributes(a, matchKind, o)
 				if err != nil {
 					wrappedErr := fmt.Errorf("failed to convert object version: %w", err)
@@ -335,68 +290,98 @@ func (c *celAdmissionController) Validate(
 				versionedAttr = va
 			}
 
-			validationResult := bindingInfo.validator.Validate(ctx, versionedAttr, param, celconfig.RuntimeCELCostBudget)
-			if err != nil {
-				// runtime error. Apply failure policy
-				wrappedError := fmt.Errorf("failed to evaluate CEL expression: %w", err)
-				addConfigError(wrappedError, definition, binding)
-				continue
+			var validationResults []ValidateResult
+			var namespace *v1.Namespace
+			namespaceName := a.GetNamespace()
+
+			// Special case, the namespace object has the namespace of itself (maybe a bug).
+			// unset it if the incoming object is a namespace
+			if gvk := a.GetKind(); gvk.Kind == "Namespace" && gvk.Version == "v1" && gvk.Group == "" {
+				namespaceName = ""
 			}
 
-			for i, decision := range validationResult.Decisions {
-				switch decision.Action {
-				case ActionAdmit:
-					if decision.Evaluation == EvalError {
-						celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
-					}
-				case ActionDeny:
-					for _, action := range binding.Spec.ValidationActions {
-						switch action {
-						case v1alpha1.Deny:
-							deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
-								Definition:     definition,
-								Binding:        binding,
-								PolicyDecision: decision,
-							})
-							celmetrics.Metrics.ObserveRejection(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
-						case v1alpha1.Audit:
-							c.publishValidationFailureAnnotation(binding, i, decision, versionedAttr)
-							celmetrics.Metrics.ObserveAudit(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
-						case v1alpha1.Warn:
-							warning.AddWarning(ctx, "", fmt.Sprintf("Validation failed for ValidatingAdmissionPolicy '%s' with binding '%s': %s", definition.Name, binding.Name, decision.Message))
-							celmetrics.Metrics.ObserveWarn(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
-						}
-					}
-				default:
-					return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'",
-						decision.Action, binding.Name, definition.Name)
+			// if it is cluster scoped, namespaceName will be empty
+			// Otherwise, get the Namespace resource.
+			if namespaceName != "" {
+				namespace, err = c.policyController.matcher.GetNamespace(namespaceName)
+				if err != nil {
+					return err
 				}
 			}
 
-			for _, auditAnnotation := range validationResult.AuditAnnotations {
-				switch auditAnnotation.Action {
-				case AuditAnnotationActionPublish:
-					value := auditAnnotation.Value
-					if len(auditAnnotation.Value) > maxAuditAnnotationValueLength {
-						value = value[:maxAuditAnnotationValueLength]
-					}
-					auditAnnotationCollector.add(auditAnnotation.Key, value)
-				case AuditAnnotationActionError:
-					// When failurePolicy=fail, audit annotation errors result in deny
-					deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
-						Definition: definition,
-						Binding:    binding,
-						PolicyDecision: PolicyDecision{
-							Action:     ActionDeny,
-							Evaluation: EvalError,
-							Message:    auditAnnotation.Error,
-							Elapsed:    auditAnnotation.Elapsed,
+			for _, param := range params {
+				var p runtime.Object = param
+				if p != nil && p.GetObjectKind().GroupVersionKind().Empty() {
+					// Make sure param has TypeMeta populated
+					// This is a simple hack to make sure typeMeta is
+					// available to CEL without making copies of objects, etc.
+					p = &wrappedParam{
+						TypeMeta: metav1.TypeMeta{
+							APIVersion: definition.Spec.ParamKind.APIVersion,
+							Kind:       definition.Spec.ParamKind.Kind,
 						},
-					})
-					celmetrics.Metrics.ObserveRejection(ctx, auditAnnotation.Elapsed, definition.Name, binding.Name, "active")
-				case AuditAnnotationActionExclude: // skip it
-				default:
-					return fmt.Errorf("unsupported AuditAnnotation Action: %s", auditAnnotation.Action)
+						nested: param,
+					}
+				}
+				validationResults = append(validationResults, bindingInfo.validator.Validate(ctx, matchResource, versionedAttr, p, namespace, celconfig.RuntimeCELCostBudget, authz))
+			}
+
+			for _, validationResult := range validationResults {
+				for i, decision := range validationResult.Decisions {
+					switch decision.Action {
+					case ActionAdmit:
+						if decision.Evaluation == EvalError {
+							celmetrics.Metrics.ObserveAdmissionWithError(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
+						}
+					case ActionDeny:
+						for _, action := range binding.Spec.ValidationActions {
+							switch action {
+							case v1beta1.Deny:
+								deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
+									Definition:     definition,
+									Binding:        binding,
+									PolicyDecision: decision,
+								})
+								celmetrics.Metrics.ObserveRejection(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
+							case v1beta1.Audit:
+								c.publishValidationFailureAnnotation(binding, i, decision, versionedAttr)
+								celmetrics.Metrics.ObserveAudit(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
+							case v1beta1.Warn:
+								warning.AddWarning(ctx, "", fmt.Sprintf("Validation failed for ValidatingAdmissionPolicy '%s' with binding '%s': %s", definition.Name, binding.Name, decision.Message))
+								celmetrics.Metrics.ObserveWarn(ctx, decision.Elapsed, definition.Name, binding.Name, "active")
+							}
+						}
+					default:
+						return fmt.Errorf("unrecognized evaluation decision '%s' for ValidatingAdmissionPolicyBinding '%s' with ValidatingAdmissionPolicy '%s'",
+							decision.Action, binding.Name, definition.Name)
+					}
+				}
+
+				for _, auditAnnotation := range validationResult.AuditAnnotations {
+					switch auditAnnotation.Action {
+					case AuditAnnotationActionPublish:
+						value := auditAnnotation.Value
+						if len(auditAnnotation.Value) > maxAuditAnnotationValueLength {
+							value = value[:maxAuditAnnotationValueLength]
+						}
+						auditAnnotationCollector.add(auditAnnotation.Key, value)
+					case AuditAnnotationActionError:
+						// When failurePolicy=fail, audit annotation errors result in deny
+						deniedDecisions = append(deniedDecisions, policyDecisionWithMetadata{
+							Definition: definition,
+							Binding:    binding,
+							PolicyDecision: PolicyDecision{
+								Action:     ActionDeny,
+								Evaluation: EvalError,
+								Message:    auditAnnotation.Error,
+								Elapsed:    auditAnnotation.Elapsed,
+							},
+						})
+						celmetrics.Metrics.ObserveRejection(ctx, auditAnnotation.Elapsed, definition.Name, binding.Name, "active")
+					case AuditAnnotationActionExclude: // skip it
+					default:
+						return fmt.Errorf("unsupported AuditAnnotation Action: %s", auditAnnotation.Action)
+					}
 				}
 			}
 		}
@@ -425,7 +410,124 @@ func (c *celAdmissionController) Validate(
 	return nil
 }
 
-func (c *celAdmissionController) publishValidationFailureAnnotation(binding *v1alpha1.ValidatingAdmissionPolicyBinding, expressionIndex int, decision PolicyDecision, attributes admission.Attributes) {
+// Returns objects to use to evaluate the policy
+func (c *celAdmissionController) collectParams(
+	paramKind *v1beta1.ParamKind,
+	info paramInfo,
+	paramRef *v1beta1.ParamRef,
+	namespace string,
+) ([]runtime.Object, error) {
+	// If definition has paramKind, paramRef is required in binding.
+	// If definition has no paramKind, paramRef set in binding will be ignored.
+	var params []runtime.Object
+	var paramStore generic.NamespacedLister[runtime.Object]
+
+	// Make sure the param kind is ready to use
+	if paramKind != nil && paramRef != nil {
+		if info.controller == nil {
+			return nil, fmt.Errorf("paramKind kind `%v` not known",
+				paramKind.String())
+		}
+
+		// Set up cluster-scoped, or namespaced access to the params
+		// "default" if not provided, and paramKind is namespaced
+		paramStore = info.controller.Informer()
+		if info.scope.Name() == meta.RESTScopeNameNamespace {
+			paramsNamespace := namespace
+			if len(paramRef.Namespace) > 0 {
+				paramsNamespace = paramRef.Namespace
+			} else if len(paramsNamespace) == 0 {
+				// You must supply namespace if your matcher can possibly
+				// match a cluster-scoped resource
+				return nil, fmt.Errorf("cannot use namespaced paramRef in policy binding that matches cluster-scoped resources")
+			}
+
+			paramStore = info.controller.Informer().Namespaced(paramsNamespace)
+		}
+
+		// If the param informer for this admission policy has not yet
+		// had time to perform an initial listing, don't attempt to use
+		// it.
+		timeoutCtx, cancel := context.WithTimeout(c.policyController.context, 1*time.Second)
+		defer cancel()
+
+		if !cache.WaitForCacheSync(timeoutCtx.Done(), info.controller.HasSynced) {
+			return nil, fmt.Errorf("paramKind kind `%v` not yet synced to use for admission",
+				paramKind.String())
+		}
+	}
+
+	// Find params to use with policy
+	switch {
+	case paramKind == nil:
+		// ParamKind is unset. Ignore any globalParamRef or namespaceParamRef
+		// setting.
+		return []runtime.Object{nil}, nil
+	case paramRef == nil:
+		// Policy ParamKind is set, but binding does not use it.
+		// Validate with nil params
+		return []runtime.Object{nil}, nil
+	case len(paramRef.Namespace) > 0 && info.scope.Name() == meta.RESTScopeRoot.Name():
+		// Not allowed to set namespace for cluster-scoped param
+		return nil, fmt.Errorf("paramRef.namespace must not be provided for a cluster-scoped `paramKind`")
+
+	case len(paramRef.Name) > 0:
+		if paramRef.Selector != nil {
+			// This should be validated, but just in case.
+			return nil, fmt.Errorf("paramRef.name and paramRef.selector are mutually exclusive")
+		}
+
+		switch param, err := paramStore.Get(paramRef.Name); {
+		case err == nil:
+			params = []runtime.Object{param}
+		case k8serrors.IsNotFound(err):
+			// Param not yet available. User may need to wait a bit
+			// before being able to use it for validation.
+			//
+			// Set params to nil to prepare for not found action
+			params = nil
+		case k8serrors.IsInvalid(err):
+			// Param mis-configured
+			// require to set namespace for namespaced resource
+			// and unset namespace for cluster scoped resource
+			return nil, err
+		default:
+			// Internal error
+			utilruntime.HandleError(err)
+			return nil, err
+		}
+	case paramRef.Selector != nil:
+		// Select everything by default if empty name and selector
+		selector, err := metav1.LabelSelectorAsSelector(paramRef.Selector)
+		if err != nil {
+			// Cannot parse label selector: configuration error
+			return nil, err
+
+		}
+
+		paramList, err := paramStore.List(selector)
+		if err != nil {
+			// There was a bad internal error
+			utilruntime.HandleError(err)
+			return nil, err
+		}
+
+		// Successfully grabbed params
+		params = paramList
+	default:
+		// Should be unreachable due to validation
+		return nil, fmt.Errorf("one of name or selector must be provided")
+	}
+
+	// Apply fail action for params not found case
+	if len(params) == 0 && paramRef.ParameterNotFoundAction != nil && *paramRef.ParameterNotFoundAction == v1beta1.DenyAction {
+		return nil, errors.New("no params found for policy binding with `Deny` parameterNotFoundAction")
+	}
+
+	return params, nil
+}
+
+func (c *celAdmissionController) publishValidationFailureAnnotation(binding *v1beta1.ValidatingAdmissionPolicyBinding, expressionIndex int, decision PolicyDecision, attributes admission.Attributes) {
 	key := "validation.policy.admission.k8s.io/validation_failure"
 	// Marshal to a list of failures since, in the future, we may need to support multiple failures
 	valueJson, err := utiljson.Marshal([]validationFailureValue{{
@@ -459,11 +561,11 @@ func (c *celAdmissionController) refreshPolicies() {
 // validationFailureValue defines the JSON format of a "validation.policy.admission.k8s.io/validation_failure" audit
 // annotation value.
 type validationFailureValue struct {
-	Message           string                      `json:"message"`
-	Policy            string                      `json:"policy"`
-	Binding           string                      `json:"binding"`
-	ExpressionIndex   int                         `json:"expressionIndex"`
-	ValidationActions []v1alpha1.ValidationAction `json:"validationActions"`
+	Message           string                     `json:"message"`
+	Policy            string                     `json:"policy"`
+	Binding           string                     `json:"binding"`
+	ExpressionIndex   int                        `json:"expressionIndex"`
+	ValidationActions []v1beta1.ValidationAction `json:"validationActions"`
 }
 
 type auditAnnotationCollector struct {
@@ -500,3 +602,48 @@ func (a auditAnnotationCollector) publish(policyName string, attributes admissio
 		}
 	}
 }
+
+// A workaround to fact that native types do not have TypeMeta populated, which
+// is needed for CEL expressions to be able to access the value.
+type wrappedParam struct {
+	metav1.TypeMeta
+	nested runtime.Object
+}
+
+func (w *wrappedParam) MarshalJSON() ([]byte, error) {
+	return nil, errors.New("MarshalJSON unimplemented for wrappedParam")
+}
+
+func (w *wrappedParam) UnmarshalJSON(data []byte) error {
+	return errors.New("UnmarshalJSON unimplemented for wrappedParam")
+}
+
+func (w *wrappedParam) ToUnstructured() interface{} {
+	res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(w.nested)
+
+	if err != nil {
+		return nil
+	}
+
+	metaRes, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&w.TypeMeta)
+	if err != nil {
+		return nil
+	}
+
+	for k, v := range metaRes {
+		res[k] = v
+	}
+
+	return res
+}
+
+func (w *wrappedParam) DeepCopyObject() runtime.Object {
+	return &wrappedParam{
+		TypeMeta: w.TypeMeta,
+		nested:   w.nested.DeepCopyObject(),
+	}
+}
+
+func (w *wrappedParam) GetObjectKind() schema.ObjectKind {
+	return w
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/controller_reconcile.go

@@ -23,11 +23,10 @@ import (
 	"time"
 
 	v1 "k8s.io/api/admissionregistration/v1"
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/meta"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -36,13 +35,11 @@ import (
 	"k8s.io/apiserver/pkg/admission/plugin/cel"
 	"k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/internal/generic"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
-	celconfig "k8s.io/apiserver/pkg/apis/cel"
-	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/cel/environment"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/dynamic/dynamicinformer"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	k8sscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/cache"
 )
 
@@ -50,36 +47,30 @@ type policyController struct {
 	once                        sync.Once
 	context                     context.Context
 	dynamicClient               dynamic.Interface
+	informerFactory             informers.SharedInformerFactory
 	restMapper                  meta.RESTMapper
-	policyDefinitionsController generic.Controller[*v1alpha1.ValidatingAdmissionPolicy]
-	policyBindingController     generic.Controller[*v1alpha1.ValidatingAdmissionPolicyBinding]
+	policyDefinitionsController generic.Controller[*v1beta1.ValidatingAdmissionPolicy]
+	policyBindingController     generic.Controller[*v1beta1.ValidatingAdmissionPolicyBinding]
 
 	// Provided to the policy's Compile function as an injected dependency to
 	// assist with compiling its expressions to CEL
+	// pass nil to create filter compiler in demand
 	filterCompiler cel.FilterCompiler
 
 	matcher Matcher
 
 	newValidator
 
-	// The TypeCheck checks the policy's expressions for type errors.
-	// Type of params is defined in policy.Spec.ParamsKind
-	// Types of object are calculated from policy.Spec.MatchingConstraints
-	typeChecker *TypeChecker
-
-	// Lock which protects:
-	//  - cachedPolicies
-	//  - paramCRDControllers
-	//  - definitionInfo
-	//  - bindingInfos
-	//  - definitionsToBindings
-	// All other fields should be assumed constant
+	client kubernetes.Interface
+	// Lock which protects
+	// All Below fields
+	// All above fields should be assumed constant
 	mutex sync.RWMutex
 
 	cachedPolicies []policyData
 
 	// controller and metadata
-	paramsCRDControllers map[v1alpha1.ParamKind]*paramInfo
+	paramsCRDControllers map[v1beta1.ParamKind]*paramInfo
 
 	// Index for each definition namespace/name, contains all binding
 	// namespace/names known to exist for that definition
@@ -94,32 +85,26 @@ type policyController struct {
 	// All keys must have at least one dependent binding
 	// All binding names MUST exist as a key bindingInfos
 	definitionsToBindings map[namespacedName]sets.Set[namespacedName]
-
-	client kubernetes.Interface
-
-	authz authorizer.Authorizer
 }
 
-type newValidator func(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failurePolicy *v1.FailurePolicyType, authorizer authorizer.Authorizer) Validator
+type newValidator func(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failurePolicy *v1.FailurePolicyType) Validator
 
 func newPolicyController(
 	restMapper meta.RESTMapper,
 	client kubernetes.Interface,
 	dynamicClient dynamic.Interface,
-	typeChecker *TypeChecker,
+	informerFactory informers.SharedInformerFactory,
 	filterCompiler cel.FilterCompiler,
 	matcher Matcher,
-	policiesInformer generic.Informer[*v1alpha1.ValidatingAdmissionPolicy],
-	bindingsInformer generic.Informer[*v1alpha1.ValidatingAdmissionPolicyBinding],
-	authz authorizer.Authorizer,
+	policiesInformer generic.Informer[*v1beta1.ValidatingAdmissionPolicy],
+	bindingsInformer generic.Informer[*v1beta1.ValidatingAdmissionPolicyBinding],
 ) *policyController {
 	res := &policyController{}
 	*res = policyController{
 		filterCompiler:        filterCompiler,
-		typeChecker:           typeChecker,
 		definitionInfo:        make(map[namespacedName]*definitionInfo),
 		bindingInfos:          make(map[namespacedName]*bindingInfo),
-		paramsCRDControllers:  make(map[v1alpha1.ParamKind]*paramInfo),
+		paramsCRDControllers:  make(map[v1beta1.ParamKind]*paramInfo),
 		definitionsToBindings: make(map[namespacedName]sets.Set[namespacedName]),
 		matcher:               matcher,
 		newValidator:          NewValidator,
@@ -139,10 +124,10 @@ func newPolicyController(
 				Name:    "cel-policy-bindings",
 			},
 		),
-		restMapper:    restMapper,
-		dynamicClient: dynamicClient,
-		client:        client,
-		authz:         authz,
+		restMapper:      restMapper,
+		dynamicClient:   dynamicClient,
+		informerFactory: informerFactory,
+		client:          client,
 	}
 	return res
 }
@@ -175,20 +160,14 @@ func (c *policyController) HasSynced() bool {
 	return c.policyDefinitionsController.HasSynced() && c.policyBindingController.HasSynced()
 }
 
-func (c *policyController) reconcilePolicyDefinition(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
+func (c *policyController) reconcilePolicyDefinition(namespace, name string, definition *v1beta1.ValidatingAdmissionPolicy) error {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 	err := c.reconcilePolicyDefinitionSpec(namespace, name, definition)
-	if err != nil {
-		return err
-	}
-	if c.typeChecker != nil {
-		err = c.reconcilePolicyStatus(namespace, name, definition)
-	}
 	return err
 }
 
-func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
+func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string, definition *v1beta1.ValidatingAdmissionPolicy) error {
 	c.cachedPolicies = nil // invalidate cachedPolicies
 
 	// Namespace for policydefinition is empty.
@@ -207,7 +186,7 @@ func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string,
 		return nil
 	}
 
-	var paramSource *v1alpha1.ParamKind
+	var paramSource *v1beta1.ParamKind
 	if definition != nil {
 		paramSource = definition.Spec.ParamKind
 	}
@@ -253,7 +232,6 @@ func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string,
 		// Skip setting up controller for empty param type
 		return nil
 	}
-
 	// find GVR for params
 	// Parse param source into a GVK
 
@@ -280,104 +258,78 @@ func (c *policyController) reconcilePolicyDefinitionSpec(namespace, name string,
 		return info.configurationError
 	}
 
-	if info, ok := c.paramsCRDControllers[*paramSource]; ok {
-		// If a param controller is already active for this paramsource, make
-		// sure it is tracking this policy's dependency upon it
-		info.dependentDefinitions.Insert(nn)
-
-	} else {
-		instanceContext, instanceCancel := context.WithCancel(c.context)
-
-		var informer cache.SharedIndexInformer
-
-		// Informer Factory is optional
-		if c.client != nil {
-			// Create temporary informer factory
-			// Cannot use the k8s shared informer factory for dynamic params informer.
-			// Would leak unnecessary informers when we are done since we would have to
-			// call informerFactory.Start() with a longer-lived stopCh than necessary.
-			// SharedInformerFactory does not support temporary usage.
-			dynamicFactory := informers.NewSharedInformerFactory(c.client, 10*time.Minute)
-
-			// Look for a typed informer. If it does not exist
-			genericInformer, err := dynamicFactory.ForResource(paramsGVR.Resource)
-
-			// Ignore error. We fallback to dynamic informer if there is no
-			// typed informer
-			if err != nil {
-				informer = nil
-			} else {
-				informer = genericInformer.Informer()
-
-				// Set transformer on the informer to workaround inconsistency
-				// where typed objects have TypeMeta wiped out but dynamic
-				// objects keep kind/apiVersion fields
-				informer.SetTransform(func(i interface{}) (interface{}, error) {
-					// Ensure param is populated with its GVK for consistency
-					// (CRD dynamic informer always returns objects with kind/apiversion,
-					// but native types do not include populated TypeMeta.
-					if param := i.(runtime.Object); param != nil {
-						if param.GetObjectKind().GroupVersionKind().Empty() {
-							// https://github.com/kubernetes/client-go/issues/413#issue-324586398
-							gvks, _, _ := k8sscheme.Scheme.ObjectKinds(param)
-							for _, gvk := range gvks {
-								if len(gvk.Kind) == 0 {
-									continue
-								}
-								if len(gvk.Version) == 0 || gvk.Version == runtime.APIVersionInternal {
-									continue
-								}
-								param.GetObjectKind().SetGroupVersionKind(gvk)
-								break
-							}
-						}
-					}
-
-					return i, nil
-				})
-			}
-		}
-
-		if informer == nil {
-			// Dynamic JSON informer fallback.
-			// Cannot use shared dynamic informer since it would be impossible
-			// to clean CRD informers properly with multiple dependents
-			// (cannot start ahead of time, and cannot track dependencies via stopCh)
-			informer = dynamicinformer.NewFilteredDynamicInformer(
-				c.dynamicClient,
-				paramsGVR.Resource,
-				corev1.NamespaceAll,
-				// Use same interval as is used for k8s typed sharedInformerFactory
-				// https://github.com/kubernetes/kubernetes/blob/7e0923899fed622efbc8679cca6b000d43633e38/cmd/kube-apiserver/app/server.go#L430
-				10*time.Minute,
-				cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
-				nil,
-			).Informer()
-		}
-
-		controller := generic.NewController(
-			generic.NewInformer[runtime.Object](informer),
-			c.reconcileParams,
-			generic.ControllerOptions{
-				Workers: 1,
-				Name:    paramSource.String() + "-controller",
-			},
-		)
-
-		c.paramsCRDControllers[*paramSource] = &paramInfo{
-			controller:           controller,
-			stop:                 instanceCancel,
-			dependentDefinitions: sets.New(nn),
-		}
-
-		go controller.Run(instanceContext)
-		go informer.Run(instanceContext.Done())
-	}
+	paramInfo := c.ensureParamInfo(paramSource, paramsGVR)
+	paramInfo.dependentDefinitions.Insert(nn)
 
 	return nil
 }
 
-func (c *policyController) reconcilePolicyBinding(namespace, name string, binding *v1alpha1.ValidatingAdmissionPolicyBinding) error {
+// Ensures that there is an informer started for the given GVK to be used as a
+// param
+func (c *policyController) ensureParamInfo(paramSource *v1beta1.ParamKind, mapping *meta.RESTMapping) *paramInfo {
+	if info, ok := c.paramsCRDControllers[*paramSource]; ok {
+		return info
+	}
+
+	// We are not watching this param. Start an informer for it.
+	instanceContext, instanceCancel := context.WithCancel(c.context)
+
+	var informer cache.SharedIndexInformer
+
+	// Try to see if our provided informer factory has an informer for this type.
+	// We assume the informer is already started, and starts all types associated
+	// with it.
+	if genericInformer, err := c.informerFactory.ForResource(mapping.Resource); err == nil {
+		informer = genericInformer.Informer()
+
+		// Ensure the informer is started
+		// Use policyController's context rather than the instance context.
+		// PolicyController context is expected to last until app shutdown
+		// This is due to behavior of informerFactory which would cause the
+		// informer to stop running once the context is cancelled, and
+		// never started again.
+		c.informerFactory.Start(c.context.Done())
+	} else {
+		// Dynamic JSON informer fallback.
+		// Cannot use shared dynamic informer since it would be impossible
+		// to clean CRD informers properly with multiple dependents
+		// (cannot start ahead of time, and cannot track dependencies via stopCh)
+		informer = dynamicinformer.NewFilteredDynamicInformer(
+			c.dynamicClient,
+			mapping.Resource,
+			corev1.NamespaceAll,
+			// Use same interval as is used for k8s typed sharedInformerFactory
+			// https://github.com/kubernetes/kubernetes/blob/7e0923899fed622efbc8679cca6b000d43633e38/cmd/kube-apiserver/app/server.go#L430
+			10*time.Minute,
+			cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc},
+			nil,
+		).Informer()
+		go informer.Run(instanceContext.Done())
+	}
+
+	controller := generic.NewController(
+		generic.NewInformer[runtime.Object](informer),
+		c.reconcileParams,
+		generic.ControllerOptions{
+			Workers: 1,
+			Name:    paramSource.String() + "-controller",
+		},
+	)
+
+	ret := &paramInfo{
+		controller:           controller,
+		stop:                 instanceCancel,
+		scope:                mapping.Scope,
+		dependentDefinitions: sets.New[namespacedName](),
+	}
+	c.paramsCRDControllers[*paramSource] = ret
+
+	go controller.Run(instanceContext)
+	return ret
+
+}
+
+func (c *policyController) reconcilePolicyBinding(namespace, name string, binding *v1beta1.ValidatingAdmissionPolicyBinding) error {
 	c.mutex.Lock()
 	defer c.mutex.Unlock()
 
@@ -443,30 +395,6 @@ func (c *policyController) reconcilePolicyBinding(namespace, name string, bindin
 	return nil
 }
 
-func (c *policyController) reconcilePolicyStatus(namespace, name string, definition *v1alpha1.ValidatingAdmissionPolicy) error {
-	if definition != nil && definition.Status.ObservedGeneration < definition.Generation {
-		st := c.calculatePolicyStatus(definition)
-		newDefinition := definition.DeepCopy()
-		newDefinition.Status = *st
-		_, err := c.client.AdmissionregistrationV1alpha1().ValidatingAdmissionPolicies().UpdateStatus(c.context, newDefinition, metav1.UpdateOptions{})
-		if err != nil {
-			// ignore error when the controller is not able to
-			// mutate the definition, and to avoid infinite requeue.
-			utilruntime.HandleError(err)
-		}
-	}
-	return nil
-}
-
-func (c *policyController) calculatePolicyStatus(definition *v1alpha1.ValidatingAdmissionPolicy) *v1alpha1.ValidatingAdmissionPolicyStatus {
-	expressionWarnings := c.typeChecker.Check(definition)
-	// modifying a deepcopy of the original status, preserving unrelated existing data
-	status := definition.Status.DeepCopy()
-	status.ObservedGeneration = definition.Generation
-	status.TypeChecking = &v1alpha1.TypeChecking{ExpressionWarnings: expressionWarnings}
-	return status
-}
-
 func (c *policyController) reconcileParams(namespace, name string, params runtime.Object) error {
 	// Do nothing.
 	// When we add informational type checking we will need to compile in the
@@ -504,39 +432,49 @@ func (c *policyController) latestPolicyData() []policyData {
 				}
 				optionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: true}
 				expressionOptionalVars := cel.OptionalVariableDeclarations{HasParams: hasParam, HasAuthorizer: false}
-				failurePolicy := convertv1alpha1FailurePolicyTypeTov1FailurePolicyType(definitionInfo.lastReconciledValue.Spec.FailurePolicy)
+				failurePolicy := convertv1beta1FailurePolicyTypeTov1FailurePolicyType(definitionInfo.lastReconciledValue.Spec.FailurePolicy)
 				var matcher matchconditions.Matcher = nil
 				matchConditions := definitionInfo.lastReconciledValue.Spec.MatchConditions
+
+				filterCompiler := c.filterCompiler
+				if filterCompiler == nil {
+					compositedCompiler, err := cel.NewCompositedCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
+					if err == nil {
+						filterCompiler = compositedCompiler
+						compositedCompiler.CompileAndStoreVariables(convertv1beta1Variables(definitionInfo.lastReconciledValue.Spec.Variables), optionalVars, environment.StoredExpressions)
+					} else {
+						utilruntime.HandleError(err)
+					}
+				}
 				if len(matchConditions) > 0 {
 					matchExpressionAccessors := make([]cel.ExpressionAccessor, len(matchConditions))
 					for i := range matchConditions {
 						matchExpressionAccessors[i] = (*matchconditions.MatchCondition)(&matchConditions[i])
 					}
-					matcher = matchconditions.NewMatcher(c.filterCompiler.Compile(matchExpressionAccessors, optionalVars, celconfig.PerCallLimit), c.authz, failurePolicy, "validatingadmissionpolicy", definitionInfo.lastReconciledValue.Name)
+					matcher = matchconditions.NewMatcher(filterCompiler.Compile(matchExpressionAccessors, optionalVars, environment.StoredExpressions), failurePolicy, "policy", "validate", definitionInfo.lastReconciledValue.Name)
 				}
 				bindingInfo.validator = c.newValidator(
-					c.filterCompiler.Compile(convertv1alpha1Validations(definitionInfo.lastReconciledValue.Spec.Validations), optionalVars, celconfig.PerCallLimit),
+					filterCompiler.Compile(convertv1beta1Validations(definitionInfo.lastReconciledValue.Spec.Validations), optionalVars, environment.StoredExpressions),
 					matcher,
-					c.filterCompiler.Compile(convertv1alpha1AuditAnnotations(definitionInfo.lastReconciledValue.Spec.AuditAnnotations), optionalVars, celconfig.PerCallLimit),
-					c.filterCompiler.Compile(convertV1Alpha1MessageExpressions(definitionInfo.lastReconciledValue.Spec.Validations), expressionOptionalVars, celconfig.PerCallLimit),
+					filterCompiler.Compile(convertv1beta1AuditAnnotations(definitionInfo.lastReconciledValue.Spec.AuditAnnotations), optionalVars, environment.StoredExpressions),
+					filterCompiler.Compile(convertv1beta1MessageExpressions(definitionInfo.lastReconciledValue.Spec.Validations), expressionOptionalVars, environment.StoredExpressions),
 					failurePolicy,
-					c.authz,
 				)
 			}
 			bindingInfos = append(bindingInfos, *bindingInfo)
 		}
 
-		var paramController generic.Controller[runtime.Object]
+		var pInfo paramInfo
 		if paramKind := definitionInfo.lastReconciledValue.Spec.ParamKind; paramKind != nil {
 			if info, ok := c.paramsCRDControllers[*paramKind]; ok {
-				paramController = info.controller
+				pInfo = *info
 			}
 		}
 
 		res = append(res, policyData{
-			definitionInfo:  *definitionInfo,
-			paramController: paramController,
-			bindings:        bindingInfos,
+			definitionInfo: *definitionInfo,
+			paramInfo:      pInfo,
+			bindings:       bindingInfos,
 		})
 	}
 
@@ -544,21 +482,21 @@ func (c *policyController) latestPolicyData() []policyData {
 	return res
 }
 
-func convertv1alpha1FailurePolicyTypeTov1FailurePolicyType(policyType *v1alpha1.FailurePolicyType) *v1.FailurePolicyType {
+func convertv1beta1FailurePolicyTypeTov1FailurePolicyType(policyType *v1beta1.FailurePolicyType) *v1.FailurePolicyType {
 	if policyType == nil {
 		return nil
 	}
 
 	var v1FailPolicy v1.FailurePolicyType
-	if *policyType == v1alpha1.Fail {
+	if *policyType == v1beta1.Fail {
 		v1FailPolicy = v1.Fail
-	} else if *policyType == v1alpha1.Ignore {
+	} else if *policyType == v1beta1.Ignore {
 		v1FailPolicy = v1.Ignore
 	}
 	return &v1FailPolicy
 }
 
-func convertv1alpha1Validations(inputValidations []v1alpha1.Validation) []cel.ExpressionAccessor {
+func convertv1beta1Validations(inputValidations []v1beta1.Validation) []cel.ExpressionAccessor {
 	celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
 	for i, validation := range inputValidations {
 		validation := ValidationCondition{
@@ -571,7 +509,7 @@ func convertv1alpha1Validations(inputValidations []v1alpha1.Validation) []cel.Ex
 	return celExpressionAccessor
 }
 
-func convertV1Alpha1MessageExpressions(inputValidations []v1alpha1.Validation) []cel.ExpressionAccessor {
+func convertv1beta1MessageExpressions(inputValidations []v1beta1.Validation) []cel.ExpressionAccessor {
 	celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
 	for i, validation := range inputValidations {
 		if validation.MessageExpression != "" {
@@ -584,7 +522,7 @@ func convertV1Alpha1MessageExpressions(inputValidations []v1alpha1.Validation) [
 	return celExpressionAccessor
 }
 
-func convertv1alpha1AuditAnnotations(inputValidations []v1alpha1.AuditAnnotation) []cel.ExpressionAccessor {
+func convertv1beta1AuditAnnotations(inputValidations []v1beta1.AuditAnnotation) []cel.ExpressionAccessor {
 	celExpressionAccessor := make([]cel.ExpressionAccessor, len(inputValidations))
 	for i, validation := range inputValidations {
 		validation := AuditAnnotationCondition{
@@ -596,6 +534,14 @@ func convertv1alpha1AuditAnnotations(inputValidations []v1alpha1.AuditAnnotation
 	return celExpressionAccessor
 }
 
+func convertv1beta1Variables(variables []v1beta1.Variable) []cel.NamedExpressionAccessor {
+	namedExpressions := make([]cel.NamedExpressionAccessor, len(variables))
+	for i, variable := range variables {
+		namedExpressions[i] = &Variable{Name: variable.Name, Expression: variable.Expression}
+	}
+	return namedExpressions
+}
+
 func getNamespaceName(namespace, name string) namespacedName {
 	return namespacedName{
 		namespace: namespace,

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/interface.go

@@ -21,12 +21,14 @@ import (
 
 	celgo "github.com/google/cel-go/cel"
 
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/plugin/cel"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
 )
 
 var _ cel.ExpressionAccessor = &ValidationCondition{}
@@ -60,17 +62,39 @@ func (v *AuditAnnotationCondition) ReturnTypes() []*celgo.Type {
 	return []*celgo.Type{celgo.StringType, celgo.NullType}
 }
 
+// Variable is a named expression for composition.
+type Variable struct {
+	Name       string
+	Expression string
+}
+
+func (v *Variable) GetExpression() string {
+	return v.Expression
+}
+
+func (v *Variable) ReturnTypes() []*celgo.Type {
+	return []*celgo.Type{celgo.AnyType, celgo.DynType}
+}
+
+func (v *Variable) GetName() string {
+	return v.Name
+}
+
 // Matcher is used for matching ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding to attributes
 type Matcher interface {
 	admission.InitializationValidator
 
 	// DefinitionMatches says whether this policy definition matches the provided admission
 	// resource request
-	DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error)
+	DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error)
 
 	// BindingMatches says whether this policy definition matches the provided admission
 	// resource request
-	BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error)
+	BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicyBinding) (bool, error)
+
+	// GetNamespace retrieves the Namespace resource by the given name. The name may be empty, in which case
+	// GetNamespace must return nil, nil
+	GetNamespace(name string) (*corev1.Namespace, error)
 }
 
 // ValidateResult defines the result of a Validator.Validate operation.
@@ -85,5 +109,5 @@ type ValidateResult struct {
 type Validator interface {
 	// Validate is used to take cel evaluations and convert into decisions
 	// runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
-	Validate(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, runtimeCELCostBudget int64) ValidateResult
+	Validate(ctx context.Context, matchedResource schema.GroupVersionResource, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *corev1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult
 }

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matcher.go

@@ -17,7 +17,8 @@ limitations under the License.
 package validatingadmissionpolicy
 
 import (
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -28,7 +29,7 @@ import (
 var _ matching.MatchCriteria = &matchCriteria{}
 
 type matchCriteria struct {
-	constraints *v1alpha1.MatchResources
+	constraints *v1beta1.MatchResources
 }
 
 // GetParsedNamespaceSelector returns the converted LabelSelector which implements labels.Selector
@@ -42,7 +43,7 @@ func (m *matchCriteria) GetParsedObjectSelector() (labels.Selector, error) {
 }
 
 // GetMatchResources returns the matchConstraints
-func (m *matchCriteria) GetMatchResources() v1alpha1.MatchResources {
+func (m *matchCriteria) GetMatchResources() v1beta1.MatchResources {
 	return *m.constraints
 }
 
@@ -62,17 +63,21 @@ func (c *matcher) ValidateInitialization() error {
 }
 
 // DefinitionMatches returns whether this ValidatingAdmissionPolicy matches the provided admission resource request
-func (c *matcher) DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1alpha1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionKind, error) {
+func (c *matcher) DefinitionMatches(a admission.Attributes, o admission.ObjectInterfaces, definition *v1beta1.ValidatingAdmissionPolicy) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
 	criteria := matchCriteria{constraints: definition.Spec.MatchConstraints}
 	return c.Matcher.Matches(a, o, &criteria)
 }
 
 // BindingMatches returns whether this ValidatingAdmissionPolicyBinding matches the provided admission resource request
-func (c *matcher) BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, binding *v1alpha1.ValidatingAdmissionPolicyBinding) (bool, error) {
+func (c *matcher) BindingMatches(a admission.Attributes, o admission.ObjectInterfaces, binding *v1beta1.ValidatingAdmissionPolicyBinding) (bool, error) {
 	if binding.Spec.MatchResources == nil {
 		return true, nil
 	}
 	criteria := matchCriteria{constraints: binding.Spec.MatchResources}
-	isMatch, _, err := c.Matcher.Matches(a, o, &criteria)
+	isMatch, _, _, err := c.Matcher.Matches(a, o, &criteria)
 	return isMatch, err
 }
+
+func (c *matcher) GetNamespace(name string) (*corev1.Namespace, error) {
+	return c.Matcher.GetNamespace(name)
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/matching/matching.go

@@ -20,7 +20,8 @@ import (
 	"fmt"
 
 	v1 "k8s.io/api/admissionregistration/v1"
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/client-go/kubernetes"
@@ -35,7 +36,7 @@ type MatchCriteria interface {
 	namespace.NamespaceSelectorProvider
 	object.ObjectSelectorProvider
 
-	GetMatchResources() v1alpha1.MatchResources
+	GetMatchResources() v1beta1.MatchResources
 }
 
 // Matcher decides if a request matches against matchCriteria
@@ -44,6 +45,10 @@ type Matcher struct {
 	objectMatcher    *object.Matcher
 }
 
+func (m *Matcher) GetNamespace(name string) (*corev1.Namespace, error) {
+	return m.namespaceMatcher.GetNamespace(name)
+}
+
 // NewMatcher initialize the matcher with dependencies requires
 func NewMatcher(
 	namespaceLister listersv1.NamespaceLister,
@@ -66,56 +71,60 @@ func (m *Matcher) ValidateInitialization() error {
 	return nil
 }
 
-func (m *Matcher) Matches(attr admission.Attributes, o admission.ObjectInterfaces, criteria MatchCriteria) (bool, schema.GroupVersionKind, error) {
+func (m *Matcher) Matches(attr admission.Attributes, o admission.ObjectInterfaces, criteria MatchCriteria) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
 	matches, matchNsErr := m.namespaceMatcher.MatchNamespaceSelector(criteria, attr)
 	// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
 	if !matches && matchNsErr == nil {
-		return false, schema.GroupVersionKind{}, nil
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
 	}
 
 	matches, matchObjErr := m.objectMatcher.MatchObjectSelector(criteria, attr)
 	// Should not return an error here for policy which do not apply to the request, even if err is an unexpected scenario.
 	if !matches && matchObjErr == nil {
-		return false, schema.GroupVersionKind{}, nil
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
 	}
 
 	matchResources := criteria.GetMatchResources()
 	matchPolicy := matchResources.MatchPolicy
-	if isExcluded, _, err := matchesResourceRules(matchResources.ExcludeResourceRules, matchPolicy, attr, o); isExcluded || err != nil {
-		return false, schema.GroupVersionKind{}, err
+	if isExcluded, _, _, err := matchesResourceRules(matchResources.ExcludeResourceRules, matchPolicy, attr, o); isExcluded || err != nil {
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, err
 	}
 
 	var (
-		isMatch   bool
-		matchKind schema.GroupVersionKind
-		matchErr  error
+		isMatch       bool
+		matchResource schema.GroupVersionResource
+		matchKind     schema.GroupVersionKind
+		matchErr      error
 	)
 	if len(matchResources.ResourceRules) == 0 {
 		isMatch = true
 		matchKind = attr.GetKind()
+		matchResource = attr.GetResource()
 	} else {
-		isMatch, matchKind, matchErr = matchesResourceRules(matchResources.ResourceRules, matchPolicy, attr, o)
+		isMatch, matchResource, matchKind, matchErr = matchesResourceRules(matchResources.ResourceRules, matchPolicy, attr, o)
 	}
 	if matchErr != nil {
-		return false, schema.GroupVersionKind{}, matchErr
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchErr
 	}
 	if !isMatch {
-		return false, schema.GroupVersionKind{}, nil
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
 	}
 
 	// now that we know this applies to this request otherwise, if there were selector errors, return them
 	if matchNsErr != nil {
-		return false, schema.GroupVersionKind{}, matchNsErr
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchNsErr
 	}
 	if matchObjErr != nil {
-		return false, schema.GroupVersionKind{}, matchObjErr
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, matchObjErr
 	}
 
-	return true, matchKind, nil
+	return true, matchResource, matchKind, nil
 }
 
-func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPolicy *v1alpha1.MatchPolicyType, attr admission.Attributes, o admission.ObjectInterfaces) (bool, schema.GroupVersionKind, error) {
+func matchesResourceRules(namedRules []v1beta1.NamedRuleWithOperations, matchPolicy *v1beta1.MatchPolicyType, attr admission.Attributes, o admission.ObjectInterfaces) (bool, schema.GroupVersionResource, schema.GroupVersionKind, error) {
 	matchKind := attr.GetKind()
+	matchResource := attr.GetResource()
+
 	for _, namedRule := range namedRules {
 		rule := v1.RuleWithOperations(namedRule.RuleWithOperations)
 		ruleMatcher := rules.Matcher{
@@ -127,22 +136,22 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
 		}
 		// an empty name list always matches
 		if len(namedRule.ResourceNames) == 0 {
-			return true, matchKind, nil
+			return true, matchResource, matchKind, nil
 		}
 		// TODO: GetName() can return an empty string if the user is relying on
 		// the API server to generate the name... figure out what to do for this edge case
 		name := attr.GetName()
 		for _, matchedName := range namedRule.ResourceNames {
 			if name == matchedName {
-				return true, matchKind, nil
+				return true, matchResource, matchKind, nil
 			}
 		}
 	}
 
 	// if match policy is undefined or exact, don't perform fuzzy matching
 	// note that defaulting to fuzzy matching is set by the API
-	if matchPolicy == nil || *matchPolicy == v1alpha1.Exact {
-		return false, schema.GroupVersionKind{}, nil
+	if matchPolicy == nil || *matchPolicy == v1beta1.Exact {
+		return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
 	}
 
 	attrWithOverride := &attrWithResourceOverride{Attributes: attr}
@@ -164,11 +173,11 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
 			}
 			matchKind = o.GetEquivalentResourceMapper().KindFor(equivalent, attr.GetSubresource())
 			if matchKind.Empty() {
-				return false, schema.GroupVersionKind{}, fmt.Errorf("unable to convert to %v: unknown kind", equivalent)
+				return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, fmt.Errorf("unable to convert to %v: unknown kind", equivalent)
 			}
 			// an empty name list always matches
 			if len(namedRule.ResourceNames) == 0 {
-				return true, matchKind, nil
+				return true, equivalent, matchKind, nil
 			}
 
 			// TODO: GetName() can return an empty string if the user is relying on
@@ -176,12 +185,12 @@ func matchesResourceRules(namedRules []v1alpha1.NamedRuleWithOperations, matchPo
 			name := attr.GetName()
 			for _, matchedName := range namedRule.ResourceNames {
 				if name == matchedName {
-					return true, matchKind, nil
+					return true, equivalent, matchKind, nil
 				}
 			}
 		}
 	}
-	return false, schema.GroupVersionKind{}, nil
+	return false, schema.GroupVersionResource{}, schema.GroupVersionKind{}, nil
 }
 
 type attrWithResourceOverride struct {

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/typechecking.go

@@ -21,19 +21,20 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
+	"time"
 
 	"github.com/google/cel-go/cel"
-	"github.com/google/cel-go/common/types/ref"
 
-	"k8s.io/api/admissionregistration/v1alpha1"
+	"k8s.io/api/admissionregistration/v1beta1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apimachinery/pkg/util/version"
 	plugincel "k8s.io/apiserver/pkg/admission/plugin/cel"
 	apiservercel "k8s.io/apiserver/pkg/cel"
 	"k8s.io/apiserver/pkg/cel/common"
+	"k8s.io/apiserver/pkg/cel/environment"
 	"k8s.io/apiserver/pkg/cel/library"
 	"k8s.io/apiserver/pkg/cel/openapi"
 	"k8s.io/apiserver/pkg/cel/openapi/resolver"
@@ -43,8 +44,17 @@ import (
 const maxTypesToCheck = 10
 
 type TypeChecker struct {
-	schemaResolver resolver.SchemaResolver
-	restMapper     meta.RESTMapper
+	SchemaResolver resolver.SchemaResolver
+	RestMapper     meta.RESTMapper
+}
+
+// TypeCheckingContext holds information about the policy being type-checked.
+// The struct is opaque to the caller.
+type TypeCheckingContext struct {
+	gvks          []schema.GroupVersionKind
+	declTypes     []*apiservercel.DeclType
+	paramGVK      schema.GroupVersionKind
+	paramDeclType *apiservercel.DeclType
 }
 
 type typeOverwrite struct {
@@ -52,127 +62,148 @@ type typeOverwrite struct {
 	params *apiservercel.DeclType
 }
 
-// typeCheckingResult holds the issues found during type checking, any returned
+// TypeCheckingResult holds the issues found during type checking, any returned
 // error, and the gvk that the type checking is performed against.
-type typeCheckingResult struct {
-	gvk schema.GroupVersionKind
+type TypeCheckingResult struct {
+	// GVK is the associated GVK
+	GVK schema.GroupVersionKind
+	// Issues contain machine-readable information about the typechecking result.
+	Issues *cel.Issues
+	// Err is the possible error that was encounter during type checking.
+	Err error
+}
 
-	issues *cel.Issues
-	err    error
+// TypeCheckingResults is a collection of TypeCheckingResult
+type TypeCheckingResults []*TypeCheckingResult
+
+func (rs TypeCheckingResults) String() string {
+	var messages []string
+	for _, r := range rs {
+		message := r.String()
+		if message != "" {
+			messages = append(messages, message)
+		}
+	}
+	return strings.Join(messages, "\n")
+}
+
+// String converts the result to human-readable form as a string.
+func (r *TypeCheckingResult) String() string {
+	if r.Issues == nil && r.Err == nil {
+		return ""
+	}
+	if r.Err != nil {
+		return fmt.Sprintf("%v: type checking error: %v\n", r.GVK, r.Err)
+	}
+	return fmt.Sprintf("%v: %s\n", r.GVK, r.Issues)
 }
 
 // Check preforms the type check against the given policy, and format the result
 // as []ExpressionWarning that is ready to be set in policy.Status
 // The result is nil if type checking returns no warning.
 // The policy object is NOT mutated. The caller should update Status accordingly
-func (c *TypeChecker) Check(policy *v1alpha1.ValidatingAdmissionPolicy) []v1alpha1.ExpressionWarning {
-	exps := make([]string, 0, len(policy.Spec.Validations))
-	// check main validation expressions, located in spec.validations[*]
+func (c *TypeChecker) Check(policy *v1beta1.ValidatingAdmissionPolicy) []v1beta1.ExpressionWarning {
+	ctx := c.CreateContext(policy)
+
+	// warnings to return, note that the capacity is optimistically set to zero
+	var warnings []v1beta1.ExpressionWarning // intentionally not setting capacity
+
+	// check main validation expressions and their message expressions, located in spec.validations[*]
 	fieldRef := field.NewPath("spec", "validations")
-	for _, v := range policy.Spec.Validations {
-		exps = append(exps, v.Expression)
-	}
-	msgs := c.CheckExpressions(exps, policy.Spec.ParamKind != nil, policy)
-	var results []v1alpha1.ExpressionWarning // intentionally not setting capacity
-	for i, msg := range msgs {
-		if msg != "" {
-			results = append(results, v1alpha1.ExpressionWarning{
+	for i, v := range policy.Spec.Validations {
+		results := c.CheckExpression(ctx, v.Expression)
+		if len(results) != 0 {
+			warnings = append(warnings, v1beta1.ExpressionWarning{
 				FieldRef: fieldRef.Index(i).Child("expression").String(),
-				Warning:  msg,
+				Warning:  results.String(),
+			})
+		}
+		// Note that MessageExpression is optional
+		if v.MessageExpression == "" {
+			continue
+		}
+		results = c.CheckExpression(ctx, v.MessageExpression)
+		if len(results) != 0 {
+			warnings = append(warnings, v1beta1.ExpressionWarning{
+				FieldRef: fieldRef.Index(i).Child("messageExpression").String(),
+				Warning:  results.String(),
 			})
 		}
 	}
-	return results
+
+	return warnings
 }
 
-// CheckExpressions checks a set of compiled CEL programs against the GVKs defined in
-// policy.Spec.MatchConstraints
-// The result is a human-readable form that describe which expressions
-// violate what types at what place. The indexes of the return []string
-// matches these of the input expressions.
-// TODO: It is much more useful to have machine-readable output and let the
-// client format it. That requires an update to the KEP, probably in coming
-// releases.
-func (c *TypeChecker) CheckExpressions(expressions []string, hasParams bool, policy *v1alpha1.ValidatingAdmissionPolicy) []string {
-	var allWarnings []string
+// CreateContext resolves all types and their schemas from a policy definition and creates the context.
+func (c *TypeChecker) CreateContext(policy *v1beta1.ValidatingAdmissionPolicy) *TypeCheckingContext {
+	ctx := new(TypeCheckingContext)
 	allGvks := c.typesToCheck(policy)
 	gvks := make([]schema.GroupVersionKind, 0, len(allGvks))
-	schemas := make([]common.Schema, 0, len(allGvks))
+	declTypes := make([]*apiservercel.DeclType, 0, len(allGvks))
 	for _, gvk := range allGvks {
-		s, err := c.schemaResolver.ResolveSchema(gvk)
+		declType, err := c.declType(gvk)
 		if err != nil {
 			// type checking errors MUST NOT alter the behavior of the policy
 			// even if an error occurs.
 			if !errors.Is(err, resolver.ErrSchemaNotFound) {
 				// Anything except ErrSchemaNotFound is an internal error
-				klog.ErrorS(err, "internal error: schema resolution failure", "gvk", gvk)
+				klog.V(2).ErrorS(err, "internal error: schema resolution failure", "gvk", gvk)
 			}
-			// skip if an unrecoverable error occurs.
+			// skip for not found or internal error
 			continue
 		}
 		gvks = append(gvks, gvk)
-		schemas = append(schemas, &openapi.Schema{Schema: s})
+		declTypes = append(declTypes, declType)
 	}
+	ctx.gvks = gvks
+	ctx.declTypes = declTypes
 
-	paramsType := c.paramsType(policy)
-	paramsDeclType, err := c.declType(paramsType)
+	paramsGVK := c.paramsGVK(policy) // maybe empty, correctly handled
+	paramsDeclType, err := c.declType(paramsGVK)
 	if err != nil {
 		if !errors.Is(err, resolver.ErrSchemaNotFound) {
-			klog.V(2).ErrorS(err, "cannot resolve schema for params", "gvk", paramsType)
+			klog.V(2).ErrorS(err, "internal error: cannot resolve schema for params", "gvk", paramsGVK)
 		}
 		paramsDeclType = nil
 	}
-
-	for _, exp := range expressions {
-		var results []typeCheckingResult
-		for i, gvk := range gvks {
-			s := schemas[i]
-			issues, err := c.checkExpression(exp, hasParams, typeOverwrite{
-				object: common.SchemaDeclType(s, true),
-				params: paramsDeclType,
-			})
-			// save even if no issues are found, for the sake of formatting.
-			results = append(results, typeCheckingResult{
-				gvk:    gvk,
-				issues: issues,
-				err:    err,
-			})
-		}
-		allWarnings = append(allWarnings, c.formatWarning(results))
-	}
-
-	return allWarnings
+	ctx.paramGVK = paramsGVK
+	ctx.paramDeclType = paramsDeclType
+	return ctx
 }
 
-// formatWarning converts the resulting issues and possible error during
-// type checking into a human-readable string
-func (c *TypeChecker) formatWarning(results []typeCheckingResult) string {
-	var sb strings.Builder
-	for _, result := range results {
-		if result.issues == nil && result.err == nil {
-			continue
-		}
-		if result.err != nil {
-			sb.WriteString(fmt.Sprintf("%v: type checking error: %v\n", result.gvk, result.err))
-		} else {
-			sb.WriteString(fmt.Sprintf("%v: %s\n", result.gvk, result.issues))
+// CheckExpression type checks a single expression, given the context
+func (c *TypeChecker) CheckExpression(ctx *TypeCheckingContext, expression string) TypeCheckingResults {
+	var results TypeCheckingResults
+	for i, gvk := range ctx.gvks {
+		declType := ctx.declTypes[i]
+		// TODO(jiahuif) hasAuthorizer always true for now, will change after expending type checking to all fields.
+		issues, err := c.checkExpression(expression, ctx.paramDeclType != nil, true, typeOverwrite{
+			object: declType,
+			params: ctx.paramDeclType,
+		})
+		if issues != nil || err != nil {
+			results = append(results, &TypeCheckingResult{Issues: issues, Err: err, GVK: gvk})
 		}
 	}
-	return strings.TrimSuffix(sb.String(), "\n")
+	return results
+}
+
+func generateUniqueTypeName(kind string) string {
+	return fmt.Sprintf("%s%d", kind, time.Now().Nanosecond())
 }
 
 func (c *TypeChecker) declType(gvk schema.GroupVersionKind) (*apiservercel.DeclType, error) {
 	if gvk.Empty() {
 		return nil, nil
 	}
-	s, err := c.schemaResolver.ResolveSchema(gvk)
+	s, err := c.SchemaResolver.ResolveSchema(gvk)
 	if err != nil {
 		return nil, err
 	}
-	return common.SchemaDeclType(&openapi.Schema{Schema: s}, true), nil
+	return common.SchemaDeclType(&openapi.Schema{Schema: s}, true).MaybeAssignTypeName(generateUniqueTypeName(gvk.Kind)), nil
 }
 
-func (c *TypeChecker) paramsType(policy *v1alpha1.ValidatingAdmissionPolicy) schema.GroupVersionKind {
+func (c *TypeChecker) paramsGVK(policy *v1beta1.ValidatingAdmissionPolicy) schema.GroupVersionKind {
 	if policy.Spec.ParamKind == nil {
 		return schema.GroupVersionKind{}
 	}
@@ -183,8 +214,8 @@ func (c *TypeChecker) paramsType(policy *v1alpha1.ValidatingAdmissionPolicy) sch
 	return gv.WithKind(policy.Spec.ParamKind.Kind)
 }
 
-func (c *TypeChecker) checkExpression(expression string, hasParams bool, types typeOverwrite) (*cel.Issues, error) {
-	env, err := buildEnv(hasParams, types)
+func (c *TypeChecker) checkExpression(expression string, hasParams, hasAuthorizer bool, types typeOverwrite) (*cel.Issues, error) {
+	env, err := buildEnv(hasParams, hasAuthorizer, types)
 	if err != nil {
 		return nil, err
 	}
@@ -202,7 +233,7 @@ func (c *TypeChecker) checkExpression(expression string, hasParams bool, types t
 
 // typesToCheck extracts a list of GVKs that needs type checking from the policy
 // the result is sorted in the order of Group, Version, and Kind
-func (c *TypeChecker) typesToCheck(p *v1alpha1.ValidatingAdmissionPolicy) []schema.GroupVersionKind {
+func (c *TypeChecker) typesToCheck(p *v1beta1.ValidatingAdmissionPolicy) []schema.GroupVersionKind {
 	gvks := sets.New[schema.GroupVersionKind]()
 	if p.Spec.MatchConstraints == nil || len(p.Spec.MatchConstraints.ResourceRules) == 0 {
 		return nil
@@ -235,7 +266,7 @@ func (c *TypeChecker) typesToCheck(p *v1alpha1.ValidatingAdmissionPolicy) []sche
 						Version:  version,
 						Resource: resource,
 					}
-					resolved, err := c.restMapper.KindsFor(gvr)
+					resolved, err := c.RestMapper.KindsFor(gvr)
 					if err != nil {
 						continue
 					}
@@ -263,7 +294,7 @@ func (c *TypeChecker) typesToCheck(p *v1alpha1.ValidatingAdmissionPolicy) []sche
 	return sortGVKList(gvks.UnsortedList())
 }
 
-func extractGroups(rule *v1alpha1.Rule) []string {
+func extractGroups(rule *v1beta1.Rule) []string {
 	groups := make([]string, 0, len(rule.APIGroups))
 	for _, group := range rule.APIGroups {
 		// give up if wildcard
@@ -275,7 +306,7 @@ func extractGroups(rule *v1alpha1.Rule) []string {
 	return groups
 }
 
-func extractVersions(rule *v1alpha1.Rule) []string {
+func extractVersions(rule *v1beta1.Rule) []string {
 	versions := make([]string, 0, len(rule.APIVersions))
 	for _, version := range rule.APIVersions {
 		if strings.ContainsAny(version, "*") {
@@ -286,7 +317,7 @@ func extractVersions(rule *v1alpha1.Rule) []string {
 	return versions
 }
 
-func extractResources(rule *v1alpha1.Rule) []string {
+func extractResources(rule *v1beta1.Rule) []string {
 	resources := make([]string, 0, len(rule.Resources))
 	for _, resource := range rule.Resources {
 		// skip wildcard and subresources
@@ -313,123 +344,64 @@ func sortGVKList(list []schema.GroupVersionKind) []schema.GroupVersionKind {
 	return list
 }
 
-func buildEnv(hasParams bool, types typeOverwrite) (*cel.Env, error) {
-	baseEnv, err := getBaseEnv()
-	if err != nil {
-		return nil, err
-	}
-	reg := apiservercel.NewRegistry(baseEnv)
+func buildEnv(hasParams bool, hasAuthorizer bool, types typeOverwrite) (*cel.Env, error) {
+	baseEnv := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())
 	requestType := plugincel.BuildRequestType()
+	namespaceType := plugincel.BuildNamespaceType()
 
 	var varOpts []cel.EnvOption
-	var rts []*apiservercel.RuleTypes
+	var declTypes []*apiservercel.DeclType
+
+	// namespace, hand-crafted type
+	declTypes = append(declTypes, namespaceType)
+	varOpts = append(varOpts, createVariableOpts(namespaceType, plugincel.NamespaceVarName)...)
 
 	// request, hand-crafted type
-	rt, opts, err := createRuleTypesAndOptions(reg, requestType, plugincel.RequestVarName)
-	if err != nil {
-		return nil, err
-	}
-	rts = append(rts, rt)
-	varOpts = append(varOpts, opts...)
+	declTypes = append(declTypes, requestType)
+	varOpts = append(varOpts, createVariableOpts(requestType, plugincel.RequestVarName)...)
 
 	// object and oldObject, same type, type(s) resolved from constraints
-	rt, opts, err = createRuleTypesAndOptions(reg, types.object, plugincel.ObjectVarName, plugincel.OldObjectVarName)
-	if err != nil {
-		return nil, err
-	}
-	rts = append(rts, rt)
-	varOpts = append(varOpts, opts...)
+	declTypes = append(declTypes, types.object)
+	varOpts = append(varOpts, createVariableOpts(types.object, plugincel.ObjectVarName, plugincel.OldObjectVarName)...)
 
 	// params, defined by ParamKind
-	if hasParams {
-		rt, opts, err := createRuleTypesAndOptions(reg, types.params, plugincel.ParamsVarName)
-		if err != nil {
-			return nil, err
-		}
-		rts = append(rts, rt)
-		varOpts = append(varOpts, opts...)
+	if hasParams && types.params != nil {
+		declTypes = append(declTypes, types.params)
+		varOpts = append(varOpts, createVariableOpts(types.params, plugincel.ParamsVarName)...)
 	}
 
-	opts, err = ruleTypesOpts(rts, baseEnv.TypeProvider())
+	// authorizer, implicitly available to all expressions of a policy
+	if hasAuthorizer {
+		// we only need its structure but not the variable itself
+		varOpts = append(varOpts, cel.Variable("authorizer", library.AuthorizerType))
+	}
+
+	env, err := baseEnv.Extend(
+		environment.VersionedOptions{
+			// Feature epoch was actually 1.26, but we artificially set it to 1.0 because these
+			// options should always be present.
+			IntroducedVersion: version.MajorMinor(1, 0),
+			EnvOptions:        varOpts,
+			DeclTypes:         declTypes,
+		},
+	)
 	if err != nil {
 		return nil, err
 	}
-	opts = append(opts, varOpts...) // add variables after ruleTypes.
-	env, err := baseEnv.Extend(opts...)
-	if err != nil {
-		return nil, err
-	}
-	return env, nil
+	return env.Env(environment.StoredExpressions)
 }
 
-// createRuleTypeAndOptions creates the cel RuleTypes and a slice of EnvOption
+// createVariableOpts creates a slice of EnvOption
 // that can be used for creating a CEL env containing variables of declType.
 // declType can be nil, in which case the variables will be of DynType.
-func createRuleTypesAndOptions(registry *apiservercel.Registry, declType *apiservercel.DeclType, variables ...string) (*apiservercel.RuleTypes, []cel.EnvOption, error) {
+func createVariableOpts(declType *apiservercel.DeclType, variables ...string) []cel.EnvOption {
 	opts := make([]cel.EnvOption, 0, len(variables))
-	// untyped, use DynType
-	if declType == nil {
-		for _, v := range variables {
-			opts = append(opts, cel.Variable(v, cel.DynType))
-		}
-		return nil, opts, nil
-	}
-	// create a RuleType for the given type
-	rt, err := apiservercel.NewRuleTypes(declType.TypeName(), declType, registry)
-	if err != nil {
-		return nil, nil, err
-	}
-	if rt == nil {
-		return nil, nil, nil
+	t := cel.DynType
+	if declType != nil {
+		t = declType.CelType()
 	}
 	for _, v := range variables {
-		opts = append(opts, cel.Variable(v, declType.CelType()))
+		opts = append(opts, cel.Variable(v, t))
 	}
-	return rt, opts, nil
+	return opts
 }
-
-func ruleTypesOpts(ruleTypes []*apiservercel.RuleTypes, underlyingTypeProvider ref.TypeProvider) ([]cel.EnvOption, error) {
-	var providers []ref.TypeProvider // may be unused, too small to matter
-	var adapters []ref.TypeAdapter
-	for _, rt := range ruleTypes {
-		if rt != nil {
-			withTP, err := rt.WithTypeProvider(underlyingTypeProvider)
-			if err != nil {
-				return nil, err
-			}
-			providers = append(providers, withTP)
-			adapters = append(adapters, withTP)
-		}
-	}
-	var tp ref.TypeProvider
-	var ta ref.TypeAdapter
-	switch len(providers) {
-	case 0:
-		return nil, nil
-	case 1:
-		tp = providers[0]
-		ta = adapters[0]
-	default:
-		tp = &apiservercel.CompositedTypeProvider{Providers: providers}
-		ta = &apiservercel.CompositedTypeAdapter{Adapters: adapters}
-	}
-	return []cel.EnvOption{cel.CustomTypeProvider(tp), cel.CustomTypeAdapter(ta)}, nil
-}
-
-func getBaseEnv() (*cel.Env, error) {
-	typeCheckingBaseEnvInit.Do(func() {
-		var opts []cel.EnvOption
-		opts = append(opts, cel.HomogeneousAggregateLiterals())
-		// Validate function declarations once during base env initialization,
-		// so they don't need to be evaluated each time a CEL rule is compiled.
-		// This is a relatively expensive operation.
-		opts = append(opts, cel.EagerlyValidateDeclarations(true), cel.DefaultUTCTimeZone(true))
-		opts = append(opts, library.ExtensionLibs...)
-		typeCheckingBaseEnv, typeCheckingBaseEnvError = cel.NewEnv(opts...)
-	})
-	return typeCheckingBaseEnv, typeCheckingBaseEnvError
-}
-
-var typeCheckingBaseEnv *cel.Env
-var typeCheckingBaseEnvError error
-var typeCheckingBaseEnvInit sync.Once

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/validator.go

@@ -24,8 +24,10 @@ import (
 	celtypes "github.com/google/cel-go/common/types"
 
 	v1 "k8s.io/api/admissionregistration/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/plugin/cel"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
@@ -42,17 +44,15 @@ type validator struct {
 	auditAnnotationFilter cel.Filter
 	messageFilter         cel.Filter
 	failPolicy            *v1.FailurePolicyType
-	authorizer            authorizer.Authorizer
 }
 
-func NewValidator(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failPolicy *v1.FailurePolicyType, authorizer authorizer.Authorizer) Validator {
+func NewValidator(validationFilter cel.Filter, celMatcher matchconditions.Matcher, auditAnnotationFilter, messageFilter cel.Filter, failPolicy *v1.FailurePolicyType) Validator {
 	return &validator{
 		celMatcher:            celMatcher,
 		validationFilter:      validationFilter,
 		auditAnnotationFilter: auditAnnotationFilter,
 		messageFilter:         messageFilter,
 		failPolicy:            failPolicy,
-		authorizer:            authorizer,
 	}
 }
 
@@ -72,7 +72,8 @@ func auditAnnotationEvaluationForError(f v1.FailurePolicyType) PolicyAuditAnnota
 
 // Validate takes a list of Evaluation and a failure policy and converts them into actionable PolicyDecisions
 // runtimeCELCostBudget was added for testing purpose only. Callers should always use const RuntimeCELCostBudget from k8s.io/apiserver/pkg/apis/cel/config.go as input.
-func (v *validator) Validate(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, runtimeCELCostBudget int64) ValidateResult {
+
+func (v *validator) Validate(ctx context.Context, matchedResource schema.GroupVersionResource, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, namespace *corev1.Namespace, runtimeCELCostBudget int64, authz authorizer.Authorizer) ValidateResult {
 	var f v1.FailurePolicyType
 	if v.failPolicy == nil {
 		f = v1.Fail
@@ -81,7 +82,7 @@ func (v *validator) Validate(ctx context.Context, versionedAttr *admission.Versi
 	}
 
 	if v.celMatcher != nil {
-		matchResults := v.celMatcher.Match(ctx, versionedAttr, versionedParams)
+		matchResults := v.celMatcher.Match(ctx, versionedAttr, versionedParams, authz)
 		if matchResults.Error != nil {
 			return ValidateResult{
 				Decisions: []PolicyDecision{
@@ -100,10 +101,12 @@ func (v *validator) Validate(ctx context.Context, versionedAttr *admission.Versi
 		}
 	}
 
-	optionalVars := cel.OptionalVariableBindings{VersionedParams: versionedParams, Authorizer: v.authorizer}
+	optionalVars := cel.OptionalVariableBindings{VersionedParams: versionedParams, Authorizer: authz}
 	expressionOptionalVars := cel.OptionalVariableBindings{VersionedParams: versionedParams}
-	admissionRequest := cel.CreateAdmissionRequest(versionedAttr.Attributes)
-	evalResults, remainingBudget, err := v.validationFilter.ForInput(ctx, versionedAttr, admissionRequest, optionalVars, runtimeCELCostBudget)
+	admissionRequest := cel.CreateAdmissionRequest(versionedAttr.Attributes, metav1.GroupVersionResource(matchedResource), metav1.GroupVersionKind(versionedAttr.VersionedKind))
+	// Decide which fields are exposed
+	ns := cel.CreateNamespaceObject(namespace)
+	evalResults, remainingBudget, err := v.validationFilter.ForInput(ctx, versionedAttr, admissionRequest, optionalVars, ns, runtimeCELCostBudget)
 	if err != nil {
 		return ValidateResult{
 			Decisions: []PolicyDecision{
@@ -116,7 +119,7 @@ func (v *validator) Validate(ctx context.Context, versionedAttr *admission.Versi
 		}
 	}
 	decisions := make([]PolicyDecision, len(evalResults))
-	messageResults, _, err := v.messageFilter.ForInput(ctx, versionedAttr, admissionRequest, expressionOptionalVars, remainingBudget)
+	messageResults, _, err := v.messageFilter.ForInput(ctx, versionedAttr, admissionRequest, expressionOptionalVars, ns, remainingBudget)
 	for i, evalResult := range evalResults {
 		var decision = &decisions[i]
 		// TODO: move this to generics
@@ -193,7 +196,7 @@ func (v *validator) Validate(ctx context.Context, versionedAttr *admission.Versi
 	}
 
 	options := cel.OptionalVariableBindings{VersionedParams: versionedParams}
-	auditAnnotationEvalResults, _, err := v.auditAnnotationFilter.ForInput(ctx, versionedAttr, cel.CreateAdmissionRequest(versionedAttr.Attributes), options, runtimeCELCostBudget)
+	auditAnnotationEvalResults, _, err := v.auditAnnotationFilter.ForInput(ctx, versionedAttr, admissionRequest, options, namespace, runtimeCELCostBudget)
 	if err != nil {
 		return ValidateResult{
 			Decisions: []PolicyDecision{

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/accessors.go

@@ -26,8 +26,7 @@ import (
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
-	celconfig "k8s.io/apiserver/pkg/apis/cel"
-	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/cel/environment"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
 	"k8s.io/client-go/rest"
 )
@@ -49,7 +48,7 @@ type WebhookAccessor interface {
 	GetRESTClient(clientManager *webhookutil.ClientManager) (*rest.RESTClient, error)
 
 	// GetCompiledMatcher gets the compiled matcher object
-	GetCompiledMatcher(compiler cel.FilterCompiler, authorizer authorizer.Authorizer) matchconditions.Matcher
+	GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher
 
 	// GetName gets the webhook Name field. Note that the name is scoped to the webhook
 	// configuration and does not provide a globally unique identity, if a unique identity is
@@ -81,6 +80,9 @@ type WebhookAccessor interface {
 	GetMutatingWebhook() (*v1.MutatingWebhook, bool)
 	// GetValidatingWebhook if the accessor contains a ValidatingWebhook, returns it and true, else returns false.
 	GetValidatingWebhook() (*v1.ValidatingWebhook, bool)
+
+	// GetType returns the type of the accessor (validate or admit)
+	GetType() string
 }
 
 // NewMutatingWebhookAccessor creates an accessor for a MutatingWebhook.
@@ -124,8 +126,11 @@ func (m *mutatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Clien
 	return m.client, m.clientErr
 }
 
-// TODO: graduation to beta: resolve the fact that we rebuild ALL items whenever ANY config changes in NewMutatingWebhookConfigurationManager and NewValidatingWebhookConfigurationManager ... now that we're doing CEL compilation, we probably want to avoid that
-func (m *mutatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler, authorizer authorizer.Authorizer) matchconditions.Matcher {
+func (m *mutatingWebhookAccessor) GetType() string {
+	return "admit"
+}
+
+func (m *mutatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
 	m.compileMatcher.Do(func() {
 		expressions := make([]cel.ExpressionAccessor, len(m.MutatingWebhook.MatchConditions))
 		for i, matchCondition := range m.MutatingWebhook.MatchConditions {
@@ -140,8 +145,8 @@ func (m *mutatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler
 				HasParams:     false,
 				HasAuthorizer: true,
 			},
-			celconfig.PerCallLimit,
-		), authorizer, m.FailurePolicy, "validating", m.Name)
+			environment.StoredExpressions,
+		), m.FailurePolicy, "webhook", "admit", m.Name)
 	})
 	return m.compiledMatcher
 }
@@ -253,7 +258,7 @@ func (v *validatingWebhookAccessor) GetRESTClient(clientManager *webhookutil.Cli
 	return v.client, v.clientErr
 }
 
-func (v *validatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler, authorizer authorizer.Authorizer) matchconditions.Matcher {
+func (v *validatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompiler) matchconditions.Matcher {
 	v.compileMatcher.Do(func() {
 		expressions := make([]cel.ExpressionAccessor, len(v.ValidatingWebhook.MatchConditions))
 		for i, matchCondition := range v.ValidatingWebhook.MatchConditions {
@@ -268,8 +273,8 @@ func (v *validatingWebhookAccessor) GetCompiledMatcher(compiler cel.FilterCompil
 				HasParams:     false,
 				HasAuthorizer: true,
 			},
-			celconfig.PerCallLimit,
-		), authorizer, v.FailurePolicy, "validating", v.Name)
+			environment.StoredExpressions,
+		), v.FailurePolicy, "webhook", "validating", v.Name)
 	})
 	return v.compiledMatcher
 }
@@ -288,6 +293,10 @@ func (v *validatingWebhookAccessor) GetParsedObjectSelector() (labels.Selector,
 	return v.objectSelector, v.objectSelectorErr
 }
 
+func (m *validatingWebhookAccessor) GetType() string {
+	return "validate"
+}
+
 func (v *validatingWebhookAccessor) GetName() string {
 	return v.Name
 }

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go

@@ -21,6 +21,9 @@ import (
 	"fmt"
 	"io"
 
+	admissionmetrics "k8s.io/apiserver/pkg/admission/metrics"
+	"k8s.io/klog/v2"
+
 	admissionv1 "k8s.io/api/admission/v1"
 	admissionv1beta1 "k8s.io/api/admission/v1beta1"
 	v1 "k8s.io/api/admissionregistration/v1"
@@ -35,10 +38,10 @@ import (
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/object"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/rules"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/cel/environment"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
 	"k8s.io/client-go/informers"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/klog/v2"
 )
 
 // Webhook is an abstract admission plugin with all the infrastructure to define Admit or Validate on-top.
@@ -97,7 +100,7 @@ func NewWebhook(handler *admission.Handler, configFile io.Reader, sourceFactory
 		namespaceMatcher: &namespace.Matcher{},
 		objectMatcher:    &object.Matcher{},
 		dispatcher:       dispatcherFactory(&cm),
-		filterCompiler:   cel.NewFilterCompiler(),
+		filterCompiler:   cel.NewFilterCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())),
 	}, nil
 }
 
@@ -216,7 +219,6 @@ func (a *Webhook) ShouldCallHook(ctx context.Context, h webhook.WebhookAccessor,
 	if matchObjErr != nil {
 		return nil, matchObjErr
 	}
-
 	matchConditions := h.GetMatchConditions()
 	if len(matchConditions) > 0 {
 		versionedAttr, err := v.VersionedAttribute(invocation.Kind)
@@ -224,13 +226,14 @@ func (a *Webhook) ShouldCallHook(ctx context.Context, h webhook.WebhookAccessor,
 			return nil, apierrors.NewInternalError(err)
 		}
 
-		matcher := h.GetCompiledMatcher(a.filterCompiler, a.authorizer)
-		matchResult := matcher.Match(ctx, versionedAttr, nil)
+		matcher := h.GetCompiledMatcher(a.filterCompiler)
+		matchResult := matcher.Match(ctx, versionedAttr, nil, a.authorizer)
 
 		if matchResult.Error != nil {
 			klog.Warningf("Failed evaluating match conditions, failing closed %v: %v", h.GetName(), matchResult.Error)
 			return nil, apierrors.NewForbidden(attr.GetResource().GroupResource(), attr.GetName(), matchResult.Error)
 		} else if !matchResult.Matches {
+			admissionmetrics.Metrics.ObserveMatchConditionExclusion(ctx, h.GetName(), "webhook", h.GetType(), string(attr.GetOperation()))
 			// if no match, always skip webhook
 			return nil, nil
 		}

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions/interface.go

@@ -21,6 +21,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/authorization/authorizer"
 )
 
 type MatchResult struct {
@@ -32,5 +33,5 @@ type MatchResult struct {
 // Matcher contains logic for converting Evaluations to bool of matches or does not match
 type Matcher interface {
 	// Match is used to take cel evaluations and convert into decisions
-	Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object) MatchResult
+	Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult
 }

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/matchconditions/matcher.go

@@ -20,11 +20,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/google/cel-go/cel"
 	celtypes "github.com/google/cel-go/common/types"
 
 	v1 "k8s.io/api/admissionregistration/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apiserver/pkg/admission"
@@ -53,13 +55,13 @@ var _ Matcher = &matcher{}
 // matcher evaluates compiled cel expressions and determines if they match the given request or not
 type matcher struct {
 	filter      celplugin.Filter
-	authorizer  authorizer.Authorizer
 	failPolicy  v1.FailurePolicyType
 	matcherType string
+	matcherKind string
 	objectName  string
 }
 
-func NewMatcher(filter celplugin.Filter, authorizer authorizer.Authorizer, failPolicy *v1.FailurePolicyType, matcherType, objectName string) Matcher {
+func NewMatcher(filter celplugin.Filter, failPolicy *v1.FailurePolicyType, matcherKind, matcherType, objectName string) Matcher {
 	var f v1.FailurePolicyType
 	if failPolicy == nil {
 		f = v1.Fail
@@ -68,20 +70,22 @@ func NewMatcher(filter celplugin.Filter, authorizer authorizer.Authorizer, failP
 	}
 	return &matcher{
 		filter:      filter,
-		authorizer:  authorizer,
 		failPolicy:  f,
+		matcherKind: matcherKind,
 		matcherType: matcherType,
 		objectName:  objectName,
 	}
 }
 
-func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object) MatchResult {
-	evalResults, _, err := m.filter.ForInput(ctx, versionedAttr, celplugin.CreateAdmissionRequest(versionedAttr.Attributes), celplugin.OptionalVariableBindings{
+func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedAttributes, versionedParams runtime.Object, authz authorizer.Authorizer) MatchResult {
+	t := time.Now()
+	evalResults, _, err := m.filter.ForInput(ctx, versionedAttr, celplugin.CreateAdmissionRequest(versionedAttr.Attributes, metav1.GroupVersionResource(versionedAttr.GetResource()), metav1.GroupVersionKind(versionedAttr.VersionedKind)), celplugin.OptionalVariableBindings{
 		VersionedParams: versionedParams,
-		Authorizer:      m.authorizer,
-	}, celconfig.RuntimeCELCostBudgetMatchConditions)
+		Authorizer:      authz,
+	}, nil, celconfig.RuntimeCELCostBudgetMatchConditions)
 
 	if err != nil {
+		admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
 		// filter returning error is unexpected and not an evaluation error so not incrementing metric here
 		if m.failPolicy == v1.Fail {
 			return MatchResult{
@@ -106,10 +110,10 @@ func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedA
 		}
 		if evalResult.Error != nil {
 			errorList = append(errorList, evalResult.Error)
-			//TODO: what's the best way to handle this metric since its reused by VAP for match conditions
-			admissionmetrics.Metrics.ObserveMatchConditionEvalError(ctx, m.objectName, m.matcherType)
+			admissionmetrics.Metrics.ObserveMatchConditionEvalError(ctx, m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
 		}
 		if evalResult.EvalResult == celtypes.False {
+			admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
 			// If any condition false, skip calling webhook always
 			return MatchResult{
 				Matches:             false,
@@ -118,6 +122,7 @@ func (m *matcher) Match(ctx context.Context, versionedAttr *admission.VersionedA
 		}
 	}
 	if len(errorList) > 0 {
+		admissionmetrics.Metrics.ObserveMatchConditionEvaluationTime(ctx, time.Since(t), m.objectName, m.matcherKind, m.matcherType, string(versionedAttr.GetOperation()))
 		// If mix of true and eval errors then resort to fail policy
 		if m.failPolicy == v1.Fail {
 			// mix of true and errors with fail policy fail should fail request without calling webhook

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go

@@ -168,6 +168,10 @@ func (a *mutatingDispatcher) Dispatch(ctx context.Context, attr admission.Attrib
 		if err != nil {
 			switch err := err.(type) {
 			case *webhookutil.ErrCallingWebhook:
+				if ctx.Err() == context.Canceled {
+					klog.Warningf("Context Canceled when calling webhook %v", hook.Name)
+					return err
+				}
 				if !ignoreClientCallFailures {
 					rejected = true
 					admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "admit", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/predicates/namespace/matcher.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 
+	v1 "k8s.io/api/core/v1"
+
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -42,6 +44,10 @@ type Matcher struct {
 	Client          clientset.Interface
 }
 
+func (m *Matcher) GetNamespace(name string) (*v1.Namespace, error) {
+	return m.NamespaceLister.Get(name)
+}
+
 // Validate checks if the Matcher has a NamespaceLister and Client.
 func (m *Matcher) Validate() error {
 	var errs []error

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go

@@ -173,6 +173,10 @@ func (d *validatingDispatcher) Dispatch(ctx context.Context, attr admission.Attr
 			if err != nil {
 				switch err := err.(type) {
 				case *webhookutil.ErrCallingWebhook:
+					if ctx.Err() == context.Canceled {
+						klog.Warningf("Context Canceled when calling webhook %v", hook.Name)
+						return
+					}
 					if !ignoreClientCallFailures {
 						rejected = true
 						admissionmetrics.Metrics.ObserveWebhookRejection(ctx, hook.Name, "validating", string(versionedAttr.Attributes.GetOperation()), admissionmetrics.WebhookRejectionCallingWebhookError, int(err.Status.ErrStatus.Code))

vendor/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go

@@ -89,6 +89,10 @@ var (
 		flowcontrol.PriorityLevelConfigurationNameExempt,
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementExempt,
+			Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{
+				NominalConcurrencyShares: pointer.Int32(0),
+				LendablePercent:          pointer.Int32(0),
+			},
 		},
 	)
 	MandatoryPriorityLevelConfigurationCatchAll = newPriorityLevelConfiguration(

vendor/k8s.io/apiserver/pkg/audit/context.go

@@ -39,21 +39,18 @@ type AuditContext struct {
 	RequestAuditConfig RequestAuditConfig
 
 	// Event is the audit Event object that is being captured to be written in
-	// the API audit log. It is set to nil when the request is not being audited.
-	Event *auditinternal.Event
+	// the API audit log.
+	Event auditinternal.Event
 
-	// annotations holds audit annotations that are recorded before the event has been initialized.
-	// This is represented as a slice rather than a map to preserve order.
-	annotations []annotation
-	// annotationMutex guards annotations AND event.Annotations
+	// annotationMutex guards event.Annotations
 	annotationMutex sync.Mutex
-
-	// auditID is the Audit ID associated with this request.
-	auditID types.UID
 }
 
-type annotation struct {
-	key, value string
+// Enabled checks whether auditing is enabled for this audit context.
+func (ac *AuditContext) Enabled() bool {
+	// Note: An unset Level should be considered Enabled, so that request data (e.g. annotations)
+	// can still be captured before the audit policy is evaluated.
+	return ac != nil && ac.RequestAuditConfig.Level != auditinternal.LevelNone
 }
 
 // AddAuditAnnotation sets the audit annotation for the given key, value pair.
@@ -65,8 +62,7 @@ type annotation struct {
 // prefer AddAuditAnnotation over LogAnnotation to avoid dropping annotations.
 func AddAuditAnnotation(ctx context.Context, key, value string) {
 	ac := AuditContextFrom(ctx)
-	if ac == nil {
-		// auditing is not enabled
+	if !ac.Enabled() {
 		return
 	}
 
@@ -81,8 +77,7 @@ func AddAuditAnnotation(ctx context.Context, key, value string) {
 // keysAndValues are the key-value pairs to add, and must have an even number of items.
 func AddAuditAnnotations(ctx context.Context, keysAndValues ...string) {
 	ac := AuditContextFrom(ctx)
-	if ac == nil {
-		// auditing is not enabled
+	if !ac.Enabled() {
 		return
 	}
 
@@ -101,8 +96,7 @@ func AddAuditAnnotations(ctx context.Context, keysAndValues ...string) {
 // restrictions on when this can be called.
 func AddAuditAnnotationsMap(ctx context.Context, annotations map[string]string) {
 	ac := AuditContextFrom(ctx)
-	if ac == nil {
-		// auditing is not enabled
+	if !ac.Enabled() {
 		return
 	}
 
@@ -114,38 +108,10 @@ func AddAuditAnnotationsMap(ctx context.Context, annotations map[string]string)
 	}
 }
 
-// addAuditAnnotationLocked is the shared code for recording an audit annotation. This method should
-// only be called while the auditAnnotationsMutex is locked.
+// addAuditAnnotationLocked records the audit annotation on the event.
 func addAuditAnnotationLocked(ac *AuditContext, key, value string) {
-	if ac.Event != nil {
-		logAnnotation(ac.Event, key, value)
-	} else {
-		ac.annotations = append(ac.annotations, annotation{key: key, value: value})
-	}
-}
+	ae := &ac.Event
 
-// This is private to prevent reads/write to the slice from outside of this package.
-// The audit event should be directly read to get access to the annotations.
-func addAuditAnnotationsFrom(ctx context.Context, ev *auditinternal.Event) {
-	ac := AuditContextFrom(ctx)
-	if ac == nil {
-		// auditing is not enabled
-		return
-	}
-
-	ac.annotationMutex.Lock()
-	defer ac.annotationMutex.Unlock()
-
-	for _, kv := range ac.annotations {
-		logAnnotation(ev, kv.key, kv.value)
-	}
-}
-
-// LogAnnotation fills in the Annotations according to the key value pair.
-func logAnnotation(ae *auditinternal.Event, key, value string) {
-	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
-		return
-	}
 	if ae.Annotations == nil {
 		ae.Annotations = make(map[string]string)
 	}
@@ -167,8 +133,8 @@ func WithAuditContext(parent context.Context) context.Context {
 
 // AuditEventFrom returns the audit event struct on the ctx
 func AuditEventFrom(ctx context.Context) *auditinternal.Event {
-	if o := AuditContextFrom(ctx); o != nil {
-		return o.Event
+	if ac := AuditContextFrom(ctx); ac.Enabled() {
+		return &ac.Event
 	}
 	return nil
 }
@@ -187,20 +153,16 @@ func WithAuditID(ctx context.Context, auditID types.UID) {
 	if auditID == "" {
 		return
 	}
-	ac := AuditContextFrom(ctx)
-	if ac == nil {
-		return
-	}
-	ac.auditID = auditID
-	if ac.Event != nil {
+	if ac := AuditContextFrom(ctx); ac != nil {
 		ac.Event.AuditID = auditID
 	}
 }
 
-// AuditIDFrom returns the value of the audit ID from the request context.
+// AuditIDFrom returns the value of the audit ID from the request context, along with whether
+// auditing is enabled.
 func AuditIDFrom(ctx context.Context) (types.UID, bool) {
 	if ac := AuditContextFrom(ctx); ac != nil {
-		return ac.auditID, ac.auditID != ""
+		return ac.Event.AuditID, true
 	}
 	return "", false
 }

vendor/k8s.io/apiserver/pkg/audit/request.go

@@ -28,14 +28,11 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/types"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/klog/v2"
-
-	"github.com/google/uuid"
 )
 
 const (
@@ -43,20 +40,18 @@ const (
 	userAgentTruncateSuffix = "...TRUNCATED"
 )
 
-func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
-	ev := &auditinternal.Event{
-		RequestReceivedTimestamp: metav1.NewMicroTime(requestReceivedTimestamp),
-		Verb:                     attribs.GetVerb(),
-		RequestURI:               req.URL.RequestURI(),
-		UserAgent:                maybeTruncateUserAgent(req),
-		Level:                    level,
+func LogRequestMetadata(ctx context.Context, req *http.Request, requestReceivedTimestamp time.Time, level auditinternal.Level, attribs authorizer.Attributes) {
+	ac := AuditContextFrom(ctx)
+	if !ac.Enabled() {
+		return
 	}
+	ev := &ac.Event
 
-	auditID, found := AuditIDFrom(req.Context())
-	if !found {
-		auditID = types.UID(uuid.New().String())
-	}
-	ev.AuditID = auditID
+	ev.RequestReceivedTimestamp = metav1.NewMicroTime(requestReceivedTimestamp)
+	ev.Verb = attribs.GetVerb()
+	ev.RequestURI = req.URL.RequestURI()
+	ev.UserAgent = maybeTruncateUserAgent(req)
+	ev.Level = level
 
 	ips := utilnet.SourceIPs(req)
 	ev.SourceIPs = make([]string, len(ips))
@@ -84,10 +79,6 @@ func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time,
 			APIVersion:  attribs.GetAPIVersion(),
 		}
 	}
-
-	addAuditAnnotationsFrom(req.Context(), ev)
-
-	return ev, nil
 }
 
 // LogImpersonatedUser fills in the impersonated user attributes into an audit event.

vendor/k8s.io/apiserver/pkg/authentication/request/websocket/protocol.go

@@ -24,8 +24,8 @@ import (
 	"strings"
 	"unicode/utf8"
 
+	"k8s.io/apimachinery/pkg/util/httpstream/wsstream"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/apiserver/pkg/util/wsstream"
 )
 
 const bearerProtocolPrefix = "base64url.bearer.authorization.k8s.io."

vendor/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go

@@ -197,15 +197,14 @@ func (a *cachedTokenAuthenticator) doAuthenticateToken(ctx context.Context, toke
 		recorder := &recorder{}
 		ctx = warning.WithWarningRecorder(ctx, recorder)
 
-		// since this is shared work between multiple requests, we have no way of knowing if any
-		// particular request supports audit annotations.  thus we always attempt to record them.
-		ev := &auditinternal.Event{Level: auditinternal.LevelMetadata}
 		ctx = audit.WithAuditContext(ctx)
 		ac := audit.AuditContextFrom(ctx)
-		ac.Event = ev
+		// since this is shared work between multiple requests, we have no way of knowing if any
+		// particular request supports audit annotations.  thus we always attempt to record them.
+		ac.Event.Level = auditinternal.LevelMetadata
 
 		record.resp, record.ok, record.err = a.authenticator.AuthenticateToken(ctx, token)
-		record.annotations = ev.Annotations
+		record.annotations = ac.Event.Annotations
 		record.warnings = recorder.extractWarnings()
 
 		if !a.cacheErrs && record.err != nil {

vendor/k8s.io/apiserver/pkg/cel/composited.go

@@ -1,119 +0,0 @@
-/*
-Copyright 2023 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cel
-
-import (
-	"github.com/google/cel-go/common/types/ref"
-	exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1"
-)
-
-var _ ref.TypeProvider = (*CompositedTypeProvider)(nil)
-var _ ref.TypeAdapter = (*CompositedTypeAdapter)(nil)
-
-// CompositedTypeProvider is the provider that tries each of the underlying
-// providers in order, and returns result of the first successful attempt.
-type CompositedTypeProvider struct {
-	// Providers contains the underlying type providers.
-	// If Providers is empty, the CompositedTypeProvider becomes no-op provider.
-	Providers []ref.TypeProvider
-}
-
-// EnumValue finds out the numeric value of the given enum name.
-// The result comes from first provider that returns non-nil.
-func (c *CompositedTypeProvider) EnumValue(enumName string) ref.Val {
-	for _, p := range c.Providers {
-		val := p.EnumValue(enumName)
-		if val != nil {
-			return val
-		}
-	}
-	return nil
-}
-
-// FindIdent takes a qualified identifier name and returns a Value if one
-// exists. The result comes from first provider that returns non-nil.
-func (c *CompositedTypeProvider) FindIdent(identName string) (ref.Val, bool) {
-	for _, p := range c.Providers {
-		val, ok := p.FindIdent(identName)
-		if ok {
-			return val, ok
-		}
-	}
-	return nil, false
-}
-
-// FindType finds the Type given a qualified type name, or return false
-// if none of the providers finds the type.
-// If any of the providers find the type, the first provider that returns true
-// will be the result.
-func (c *CompositedTypeProvider) FindType(typeName string) (*exprpb.Type, bool) {
-	for _, p := range c.Providers {
-		typ, ok := p.FindType(typeName)
-		if ok {
-			return typ, ok
-		}
-	}
-	return nil, false
-}
-
-// FindFieldType returns the field type for a checked type value. Returns
-// false if none of the providers can find the type.
-// If multiple providers can find the field, the result is taken from
-// the first that does.
-func (c *CompositedTypeProvider) FindFieldType(messageType string, fieldName string) (*ref.FieldType, bool) {
-	for _, p := range c.Providers {
-		ft, ok := p.FindFieldType(messageType, fieldName)
-		if ok {
-			return ft, ok
-		}
-	}
-	return nil, false
-}
-
-// NewValue creates a new type value from a qualified name and map of field
-// name to value.
-// If multiple providers can create the new type, the first that returns
-// non-nil will decide the result.
-func (c *CompositedTypeProvider) NewValue(typeName string, fields map[string]ref.Val) ref.Val {
-	for _, p := range c.Providers {
-		v := p.NewValue(typeName, fields)
-		if v != nil {
-			return v
-		}
-	}
-	return nil
-}
-
-// CompositedTypeAdapter is the adapter that tries each of the underlying
-// type adapter in order until the first successfully conversion.
-type CompositedTypeAdapter struct {
-	// Adapters contains underlying type adapters.
-	// If Adapters is empty, the CompositedTypeAdapter becomes a no-op adapter.
-	Adapters []ref.TypeAdapter
-}
-
-// NativeToValue takes the value and convert it into a ref.Val
-// The result comes from the first TypeAdapter that returns non-nil.
-func (c *CompositedTypeAdapter) NativeToValue(value interface{}) ref.Val {
-	for _, a := range c.Adapters {
-		v := a.NativeToValue(value)
-		if v != nil {
-			return v
-		}
-	}
-	return nil
-}

vendor/k8s.io/apiserver/pkg/cel/environment/base.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package environment
+
+import (
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/ext"
+	"golang.org/x/sync/singleflight"
+
+	"k8s.io/apimachinery/pkg/util/version"
+	celconfig "k8s.io/apiserver/pkg/apis/cel"
+	"k8s.io/apiserver/pkg/cel/library"
+)
+
+// DefaultCompatibilityVersion returns a default compatibility version for use with EnvSet
+// that guarantees compatibility with CEL features/libraries/parameters understood by
+// an n-1 version
+//
+// This default will be set to no more than n-1 the current Kubernetes major.minor version.
+//
+// Note that a default version number less than n-1 indicates a wider range of version
+// compatibility than strictly required for rollback. A wide range of compatibility is
+// desirable because it means that CEL expressions are portable across a wider range
+// of Kubernetes versions.
+func DefaultCompatibilityVersion() *version.Version {
+	return version.MajorMinor(1, 27)
+}
+
+var baseOpts = []VersionedOptions{
+	{
+		// CEL epoch was actually 1.23, but we artificially set it to 1.0 because these
+		// options should always be present.
+		IntroducedVersion: version.MajorMinor(1, 0),
+		EnvOptions: []cel.EnvOption{
+			cel.HomogeneousAggregateLiterals(),
+			// Validate function declarations once during base env initialization,
+			// so they don't need to be evaluated each time a CEL rule is compiled.
+			// This is a relatively expensive operation.
+			cel.EagerlyValidateDeclarations(true),
+			cel.DefaultUTCTimeZone(true),
+
+			ext.Strings(ext.StringsVersion(0)),
+			library.URLs(),
+			library.Regex(),
+			library.Lists(),
+		},
+		ProgramOptions: []cel.ProgramOption{
+			cel.EvalOptions(cel.OptOptimize, cel.OptTrackCost),
+			cel.CostLimit(celconfig.PerCallLimit),
+		},
+	},
+	{
+		IntroducedVersion: version.MajorMinor(1, 27),
+		EnvOptions: []cel.EnvOption{
+			library.Authz(),
+		},
+	},
+	{
+		IntroducedVersion: version.MajorMinor(1, 28),
+		EnvOptions: []cel.EnvOption{
+			cel.CrossTypeNumericComparisons(true),
+			cel.OptionalTypes(),
+			library.Quantity(),
+		},
+	},
+	// TODO: switch to ext.Strings version 2 once format() is fixed to work with HomogeneousAggregateLiterals.
+}
+
+// MustBaseEnvSet returns the common CEL base environments for Kubernetes for Version, or panics
+// if the version is nil, or does not have major and minor components.
+//
+// The returned environment contains function libraries, language settings, optimizations and
+// runtime cost limits appropriate CEL as it is used in Kubernetes.
+//
+// The returned environment contains no CEL variable definitions or custom type declarations and
+// should be extended to construct environments with the appropriate variable definitions,
+// type declarations and any other needed configuration.
+func MustBaseEnvSet(ver *version.Version) *EnvSet {
+	if ver == nil {
+		panic("version must be non-nil")
+	}
+	if len(ver.Components()) < 2 {
+		panic(fmt.Sprintf("version must contain an major and minor component, but got: %s", ver.String()))
+	}
+	key := strconv.FormatUint(uint64(ver.Major()), 10) + "." + strconv.FormatUint(uint64(ver.Minor()), 10)
+	if entry, ok := baseEnvs.Load(key); ok {
+		return entry.(*EnvSet)
+	}
+
+	entry, _, _ := baseEnvsSingleflight.Do(key, func() (interface{}, error) {
+		entry := mustNewEnvSet(ver, baseOpts)
+		baseEnvs.Store(key, entry)
+		return entry, nil
+	})
+	return entry.(*EnvSet)
+}
+
+var (
+	baseEnvs             = sync.Map{}
+	baseEnvsSingleflight = &singleflight.Group{}
+)

vendor/k8s.io/apiserver/pkg/cel/environment/environment.go

@@ -0,0 +1,274 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package environment
+
+import (
+	"fmt"
+	"math"
+
+	"github.com/google/cel-go/cel"
+
+	"k8s.io/apimachinery/pkg/util/version"
+	apiservercel "k8s.io/apiserver/pkg/cel"
+)
+
+// Type defines the different types of CEL environments used in Kubernetes.
+// CEL environments are used to compile and evaluate CEL expressions.
+// Environments include:
+//   - Function libraries
+//   - Variables
+//   - Types (both core CEL types and Kubernetes types)
+//   - Other CEL environment and program options
+type Type string
+
+const (
+	// NewExpressions is used to validate new or modified expressions in
+	// requests that write expressions to API resources.
+	//
+	// This environment type is compatible with a specific Kubernetes
+	// major/minor version. To ensure safe rollback, this environment type
+	// may not include all the function libraries, variables, type declarations, and CEL
+	// language settings available in the StoredExpressions environment type.
+	//
+	// NewExpressions must be used to validate (parse, compile, type check)
+	// all new or modified CEL expressions before they are written to storage.
+	NewExpressions Type = "NewExpressions"
+
+	// StoredExpressions is used to compile and run CEL expressions that have been
+	// persisted to storage.
+	//
+	// This environment type is compatible with CEL expressions that have been
+	// persisted to storage by all known versions of Kubernetes. This is the most
+	// permissive environment available.
+	//
+	// StoredExpressions is appropriate for use with CEL expressions in
+	// configuration files.
+	StoredExpressions Type = "StoredExpressions"
+)
+
+// EnvSet manages the creation and extension of CEL environments. Each EnvSet contains
+// both an NewExpressions and StoredExpressions environment. EnvSets are created
+// and extended using VersionedOptions so that the EnvSet can prepare environments according
+// to what options were introduced at which versions.
+//
+// Each EnvSet is given a compatibility version when it is created, and prepares the
+// NewExpressions environment to be compatible with that version. The EnvSet also
+// prepares StoredExpressions to be compatible with all known versions of Kubernetes.
+type EnvSet struct {
+	// compatibilityVersion is the version that all configuration in
+	// the NewExpressions environment is compatible with.
+	compatibilityVersion *version.Version
+
+	// newExpressions is an environment containing only configuration
+	// in this EnvSet that is enabled at this compatibilityVersion.
+	newExpressions *cel.Env
+
+	// storedExpressions is an environment containing the latest configuration
+	// in this EnvSet.
+	storedExpressions *cel.Env
+}
+
+func newEnvSet(compatibilityVersion *version.Version, opts []VersionedOptions) (*EnvSet, error) {
+	base, err := cel.NewEnv()
+	if err != nil {
+		return nil, err
+	}
+	baseSet := EnvSet{compatibilityVersion: compatibilityVersion, newExpressions: base, storedExpressions: base}
+	return baseSet.Extend(opts...)
+}
+
+func mustNewEnvSet(ver *version.Version, opts []VersionedOptions) *EnvSet {
+	envSet, err := newEnvSet(ver, opts)
+	if err != nil {
+		panic(fmt.Sprintf("Default environment misconfigured: %v", err))
+	}
+	return envSet
+}
+
+// NewExpressionsEnv returns the NewExpressions environment Type for this EnvSet.
+// See NewExpressions for details.
+func (e *EnvSet) NewExpressionsEnv() *cel.Env {
+	return e.newExpressions
+}
+
+// StoredExpressionsEnv returns the StoredExpressions environment Type for this EnvSet.
+// See StoredExpressions for details.
+func (e *EnvSet) StoredExpressionsEnv() *cel.Env {
+	return e.storedExpressions
+}
+
+// Env returns the CEL environment for the given Type.
+func (e *EnvSet) Env(envType Type) (*cel.Env, error) {
+	switch envType {
+	case NewExpressions:
+		return e.newExpressions, nil
+	case StoredExpressions:
+		return e.storedExpressions, nil
+	default:
+		return nil, fmt.Errorf("unsupported environment type: %v", envType)
+	}
+}
+
+// VersionedOptions provides a set of CEL configuration options as well as the version the
+// options were introduced and, optionally, the version the options were removed.
+type VersionedOptions struct {
+	// IntroducedVersion is the version at which these options were introduced.
+	// The NewExpressions environment will only include options introduced at or before the
+	// compatibility version of the EnvSet.
+	//
+	// For example, to configure a CEL environment with an "object" variable bound to a
+	// resource kind, first create a DeclType from the groupVersionKind of the resource and then
+	// populate a VersionedOptions with the variable and the type:
+	//
+	//    schema := schemaResolver.ResolveSchema(groupVersionKind)
+	//    objectType := apiservercel.SchemaDeclType(schema, true)
+	//    ...
+	//    VersionOptions{
+	//      IntroducedVersion: version.MajorMinor(1, 26),
+	//      DeclTypes: []*apiservercel.DeclType{ objectType },
+	//      EnvOptions: []cel.EnvOption{ cel.Variable("object", objectType.CelType()) },
+	//    },
+	//
+	// To create an DeclType from a CRD, use a structural schema. For example:
+	//
+	//    schema := structuralschema.NewStructural(crdJSONProps)
+	//    objectType := apiservercel.SchemaDeclType(schema, true)
+	//
+	// Required.
+	IntroducedVersion *version.Version
+	// RemovedVersion is the version at which these options were removed.
+	// The NewExpressions environment will not include options removed at or before the
+	// compatibility version of the EnvSet.
+	//
+	// All option removals must be backward compatible; the removal must either be paired
+	// with a compatible replacement introduced at the same version, or the removal must be non-breaking.
+	// The StoredExpressions environment will not include removed options.
+	//
+	// A function library may be upgraded by setting the RemovedVersion of the old library
+	// to the same value as the IntroducedVersion of the new library. The new library must
+	// be backward compatible with the old library.
+	//
+	// For example:
+	//
+	//    VersionOptions{
+	//      IntroducedVersion: version.MajorMinor(1, 26), RemovedVersion: version.MajorMinor(1, 27),
+	//      EnvOptions: []cel.EnvOption{ libraries.Example(libraries.ExampleVersion(1)) },
+	//    },
+	//    VersionOptions{
+	//      IntroducedVersion: version.MajorMinor(1, 27),
+	//      EnvOptions: []EnvOptions{ libraries.Example(libraries.ExampleVersion(2)) },
+	//    },
+	//
+	// Optional.
+	RemovedVersion *version.Version
+
+	// EnvOptions provides CEL EnvOptions. This may be used to add a cel.Variable, a
+	// cel.Library, or to enable other CEL EnvOptions such as language settings.
+	//
+	// If an added cel.Variable has an OpenAPI type, the type must be included in DeclTypes.
+	EnvOptions []cel.EnvOption
+	// ProgramOptions provides CEL ProgramOptions. This may be used to set a cel.CostLimit,
+	// enable optimizations, and set other program level options that should be enabled
+	// for all programs using this environment.
+	ProgramOptions []cel.ProgramOption
+	// DeclTypes provides OpenAPI type declarations to register with the environment.
+	//
+	// If cel.Variables added to EnvOptions refer to a OpenAPI type, the type must be included in
+	// DeclTypes.
+	DeclTypes []*apiservercel.DeclType
+}
+
+// Extend returns an EnvSet based on this EnvSet but extended with given VersionedOptions.
+// This EnvSet is not mutated.
+// The returned EnvSet has the same compatibility version as the EnvSet that was extended.
+//
+// Extend is an expensive operation and each call to Extend that adds DeclTypes increases
+// the depth of a chain of resolvers. For these reasons, calls to Extend should be kept
+// to a minimum.
+//
+// Some best practices:
+//
+//   - Minimize calls Extend when handling API requests. Where possible, call Extend
+//     when initializing components.
+//   - If an EnvSets returned by Extend can be used to compile multiple CEL programs,
+//     call Extend once and reuse the returned EnvSets.
+//   - Prefer a single call to Extend with a full list of VersionedOptions over
+//     making multiple calls to Extend.
+func (e *EnvSet) Extend(options ...VersionedOptions) (*EnvSet, error) {
+	if len(options) > 0 {
+		newExprOpts, err := e.filterAndBuildOpts(e.newExpressions, e.compatibilityVersion, options)
+		if err != nil {
+			return nil, err
+		}
+		p, err := e.newExpressions.Extend(newExprOpts)
+		if err != nil {
+			return nil, err
+		}
+		storedExprOpt, err := e.filterAndBuildOpts(e.storedExpressions, version.MajorMinor(math.MaxUint, math.MaxUint), options)
+		if err != nil {
+			return nil, err
+		}
+		s, err := e.storedExpressions.Extend(storedExprOpt)
+		if err != nil {
+			return nil, err
+		}
+		return &EnvSet{compatibilityVersion: e.compatibilityVersion, newExpressions: p, storedExpressions: s}, nil
+	}
+	return e, nil
+}
+
+func (e *EnvSet) filterAndBuildOpts(base *cel.Env, compatVer *version.Version, opts []VersionedOptions) (cel.EnvOption, error) {
+	var envOpts []cel.EnvOption
+	var progOpts []cel.ProgramOption
+	var declTypes []*apiservercel.DeclType
+
+	for _, opt := range opts {
+		if compatVer.AtLeast(opt.IntroducedVersion) && (opt.RemovedVersion == nil || compatVer.LessThan(opt.RemovedVersion)) {
+			envOpts = append(envOpts, opt.EnvOptions...)
+			progOpts = append(progOpts, opt.ProgramOptions...)
+			declTypes = append(declTypes, opt.DeclTypes...)
+		}
+	}
+
+	if len(declTypes) > 0 {
+		provider := apiservercel.NewDeclTypeProvider(declTypes...)
+		providerOpts, err := provider.EnvOptions(base.TypeProvider())
+		if err != nil {
+			return nil, err
+		}
+		envOpts = append(envOpts, providerOpts...)
+	}
+
+	combined := cel.Lib(&envLoader{
+		envOpts:  envOpts,
+		progOpts: progOpts,
+	})
+	return combined, nil
+}
+
+type envLoader struct {
+	envOpts  []cel.EnvOption
+	progOpts []cel.ProgramOption
+}
+
+func (e *envLoader) CompileOptions() []cel.EnvOption {
+	return e.envOpts
+}
+
+func (e *envLoader) ProgramOptions() []cel.ProgramOption {
+	return e.progOpts
+}

vendor/k8s.io/apiserver/pkg/cel/lazy/lazy.go

@@ -0,0 +1,191 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lazy
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/common/types/traits"
+
+	"k8s.io/apiserver/pkg/cel"
+)
+
+type GetFieldFunc func(*MapValue) ref.Val
+
+var _ ref.Val = (*MapValue)(nil)
+var _ traits.Mapper = (*MapValue)(nil)
+
+// MapValue is a map that lazily evaluate its value when a field is first accessed.
+// The map value is not designed to be thread-safe.
+type MapValue struct {
+	typeValue *types.TypeValue
+
+	// values are previously evaluated values obtained from callbacks
+	values map[string]ref.Val
+	// callbacks are a map of field name to the function that returns the field Val
+	callbacks map[string]GetFieldFunc
+	// knownValues are registered names, used for iteration
+	knownValues []string
+}
+
+func NewMapValue(objectType ref.Type) *MapValue {
+	return &MapValue{
+		typeValue: types.NewTypeValue(objectType.TypeName(), traits.IndexerType|traits.FieldTesterType|traits.IterableType),
+		values:    map[string]ref.Val{},
+		callbacks: map[string]GetFieldFunc{},
+	}
+}
+
+// Append adds the given field with its name and callback.
+func (m *MapValue) Append(name string, callback GetFieldFunc) {
+	m.knownValues = append(m.knownValues, name)
+	m.callbacks[name] = callback
+}
+
+// Contains checks if the key is known to the map
+func (m *MapValue) Contains(key ref.Val) ref.Val {
+	v, found := m.Find(key)
+	if v != nil && types.IsUnknownOrError(v) {
+		return v
+	}
+	return types.Bool(found)
+}
+
+// Iterator returns an iterator to traverse the map.
+func (m *MapValue) Iterator() traits.Iterator {
+	return &iterator{parent: m, index: 0}
+}
+
+// Size returns the number of currently known fields
+func (m *MapValue) Size() ref.Val {
+	return types.Int(len(m.callbacks))
+}
+
+// ConvertToNative returns an error because it is disallowed
+func (m *MapValue) ConvertToNative(typeDesc reflect.Type) (any, error) {
+	return nil, fmt.Errorf("disallowed conversion from %q to %q", m.typeValue.TypeName(), typeDesc.Name())
+}
+
+// ConvertToType converts the map to the given type.
+// Only its own type and "Type" type are allowed.
+func (m *MapValue) ConvertToType(typeVal ref.Type) ref.Val {
+	switch typeVal {
+	case m.typeValue:
+		return m
+	case types.TypeType:
+		return m.typeValue
+	}
+	return types.NewErr("disallowed conversion from %q to %q", m.typeValue.TypeName(), typeVal.TypeName())
+}
+
+// Equal returns true if the other object is the same pointer-wise.
+func (m *MapValue) Equal(other ref.Val) ref.Val {
+	otherMap, ok := other.(*MapValue)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(other)
+	}
+	return types.Bool(m == otherMap)
+}
+
+// Type returns its registered type.
+func (m *MapValue) Type() ref.Type {
+	return m.typeValue
+}
+
+// Value is not allowed.
+func (m *MapValue) Value() any {
+	return types.NoSuchOverloadErr()
+}
+
+// resolveField resolves the field. Calls the callback if the value is not yet stored.
+func (m *MapValue) resolveField(name string) ref.Val {
+	v, seen := m.values[name]
+	if seen {
+		return v
+	}
+	f := m.callbacks[name]
+	v = f(m)
+	m.values[name] = v
+	return v
+}
+
+func (m *MapValue) Find(key ref.Val) (ref.Val, bool) {
+	n, ok := key.(types.String)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(n), true
+	}
+	name, ok := cel.Unescape(n.Value().(string))
+	if !ok {
+		return nil, false
+	}
+	if _, exists := m.callbacks[name]; !exists {
+		return nil, false
+	}
+	return m.resolveField(name), true
+}
+
+func (m *MapValue) Get(key ref.Val) ref.Val {
+	v, found := m.Find(key)
+	if found {
+		return v
+	}
+	return types.ValOrErr(key, "no such key: %v", key)
+}
+
+type iterator struct {
+	parent *MapValue
+	index  int
+}
+
+func (i *iterator) ConvertToNative(typeDesc reflect.Type) (any, error) {
+	return nil, fmt.Errorf("disallowed conversion to %q", typeDesc.Name())
+}
+
+func (i *iterator) ConvertToType(typeValue ref.Type) ref.Val {
+	return types.NewErr("disallowed conversion o %q", typeValue.TypeName())
+}
+
+func (i *iterator) Equal(other ref.Val) ref.Val {
+	otherIterator, ok := other.(*iterator)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(other)
+	}
+	return types.Bool(otherIterator == i)
+}
+
+func (i *iterator) Type() ref.Type {
+	return types.IteratorType
+}
+
+func (i *iterator) Value() any {
+	return nil
+}
+
+func (i *iterator) HasNext() ref.Val {
+	return types.Bool(i.index < len(i.parent.knownValues))
+}
+
+func (i *iterator) Next() ref.Val {
+	ret := i.parent.Get(types.String(i.parent.knownValues[i.index]))
+	i.index++
+	return ret
+}
+
+var _ traits.Iterator = (*iterator)(nil)

vendor/k8s.io/apiserver/pkg/cel/library/authz.go

@@ -174,6 +174,26 @@ import (
 // Examples:
 //
 //	authorizer.path('/healthz').check('GET').reason()
+//
+// errored
+//
+// Returns true if the authorization check resulted in an error.
+//
+//	<Decision>.errored() <bool>
+//
+// Examples:
+//
+//	authorizer.group('').resource('pods').namespace('default').check('create').errored() // Returns true if the authorization check resulted in an error
+//
+// error
+//
+// If the authorization check resulted in an error, returns the error. Otherwise, returns the empty string.
+//
+//	<Decision>.error() <string>
+//
+// Examples:
+//
+//	authorizer.group('').resource('pods').namespace('default').check('create').error()
 func Authz() cel.EnvOption {
 	return cel.Lib(authzLib)
 }
@@ -209,6 +229,12 @@ var authzLibraryDecls = map[string][]cel.FunctionOpt{
 			cel.BinaryBinding(pathCheckCheck)),
 		cel.MemberOverload("resourcecheck_check", []*cel.Type{ResourceCheckType, cel.StringType}, DecisionType,
 			cel.BinaryBinding(resourceCheckCheck))},
+	"errored": {
+		cel.MemberOverload("decision_errored", []*cel.Type{DecisionType}, cel.BoolType,
+			cel.UnaryBinding(decisionErrored))},
+	"error": {
+		cel.MemberOverload("decision_error", []*cel.Type{DecisionType}, cel.StringType,
+			cel.UnaryBinding(decisionError))},
 	"allowed": {
 		cel.MemberOverload("decision_allowed", []*cel.Type{DecisionType}, cel.BoolType,
 			cel.UnaryBinding(decisionAllowed))},
@@ -384,6 +410,27 @@ func resourceCheckCheck(arg1, arg2 ref.Val) ref.Val {
 	return resourceCheck.Authorize(context.TODO(), apiVerb)
 }
 
+func decisionErrored(arg ref.Val) ref.Val {
+	decision, ok := arg.(decisionVal)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	return types.Bool(decision.err != nil)
+}
+
+func decisionError(arg ref.Val) ref.Val {
+	decision, ok := arg.(decisionVal)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	if decision.err == nil {
+		return types.String("")
+	}
+	return types.String(decision.err.Error())
+}
+
 func decisionAllowed(arg ref.Val) ref.Val {
 	decision, ok := arg.(decisionVal)
 	if !ok {
@@ -478,10 +525,7 @@ func (a pathCheckVal) Authorize(ctx context.Context, verb string) ref.Val {
 	}
 
 	decision, reason, err := a.authorizer.authAuthorizer.Authorize(ctx, attr)
-	if err != nil {
-		return types.NewErr("error in authorization check: %v", err)
-	}
-	return newDecision(decision, reason)
+	return newDecision(decision, err, reason)
 }
 
 type groupCheckVal struct {
@@ -516,18 +560,16 @@ func (a resourceCheckVal) Authorize(ctx context.Context, verb string) ref.Val {
 		User:            a.groupCheck.authorizer.userInfo,
 	}
 	decision, reason, err := a.groupCheck.authorizer.authAuthorizer.Authorize(ctx, attr)
-	if err != nil {
-		return types.NewErr("error in authorization check: %v", err)
-	}
-	return newDecision(decision, reason)
+	return newDecision(decision, err, reason)
 }
 
-func newDecision(authDecision authorizer.Decision, reason string) decisionVal {
-	return decisionVal{receiverOnlyObjectVal: receiverOnlyVal(DecisionType), authDecision: authDecision, reason: reason}
+func newDecision(authDecision authorizer.Decision, err error, reason string) decisionVal {
+	return decisionVal{receiverOnlyObjectVal: receiverOnlyVal(DecisionType), authDecision: authDecision, err: err, reason: reason}
 }
 
 type decisionVal struct {
 	receiverOnlyObjectVal
+	err          error
 	authDecision authorizer.Decision
 	reason       string
 }

vendor/k8s.io/apiserver/pkg/cel/library/cost.go

@@ -41,7 +41,7 @@ func (l *CostEstimator) CallCost(function, overloadId string, args []ref.Val, re
 		// This cost is set to allow for only two authorization checks per expression
 		cost := uint64(350000)
 		return &cost
-	case "serviceAccount", "path", "group", "resource", "subresource", "namespace", "name", "allowed", "denied", "reason":
+	case "serviceAccount", "path", "group", "resource", "subresource", "namespace", "name", "allowed", "reason", "error", "errored":
 		// All authorization builder and accessor functions have a nominal cost
 		cost := uint64(1)
 		return &cost
@@ -91,7 +91,7 @@ func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *ch
 		// An authorization check has a fixed cost
 		// This cost is set to allow for only two authorization checks per expression
 		return &checker.CallEstimate{CostEstimate: checker.CostEstimate{Min: 350000, Max: 350000}}
-	case "serviceAccount", "path", "group", "resource", "subresource", "namespace", "name", "allowed", "denied", "reason":
+	case "serviceAccount", "path", "group", "resource", "subresource", "namespace", "name", "allowed", "reason", "error", "errored":
 		// All authorization builder and accessor functions have a nominal cost
 		return &checker.CallEstimate{CostEstimate: checker.CostEstimate{Min: 1, Max: 1}}
 	case "isSorted", "sum", "max", "min", "indexOf", "lastIndexOf":

vendor/k8s.io/apiserver/pkg/cel/library/libraries.go

@@ -1,35 +0,0 @@
-/*
-Copyright 2022 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package library
-
-import (
-	"github.com/google/cel-go/cel"
-	"github.com/google/cel-go/ext"
-	"github.com/google/cel-go/interpreter"
-)
-
-// ExtensionLibs declares the set of CEL extension libraries available everywhere CEL is used in Kubernetes.
-var ExtensionLibs = append(k8sExtensionLibs, ext.Strings())
-
-var k8sExtensionLibs = []cel.EnvOption{
-	URLs(),
-	Regex(),
-	Lists(),
-	Authz(),
-}
-
-var ExtensionLibRegexOptimizations = []*interpreter.RegexOptimization{FindRegexOptimization, FindAllRegexOptimization}

vendor/k8s.io/apiserver/pkg/cel/library/quantity.go

@@ -0,0 +1,375 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package library
+
+import (
+	"errors"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"k8s.io/apimachinery/pkg/api/resource"
+	apiservercel "k8s.io/apiserver/pkg/cel"
+)
+
+// Quantity provides a CEL function library extension of Kubernetes
+// resource.Quantity parsing functions. See `resource.Quantity`
+// documentation for more detailed information about the format itself:
+// https://pkg.go.dev/k8s.io/apimachinery/pkg/api/resource#Quantity
+//
+// quantity
+//
+// Converts a string to a Quantity or results in an error if the string is not a valid Quantity. Refer
+// to resource.Quantity documentation for information on accepted patterns.
+//
+//	quantity(<string>) <Quantity>
+//
+// Examples:
+//
+//	quantity('1.5G') // returns a Quantity
+//	quantity('200k') // returns a Quantity
+//	quantity('200K') // error
+//	quantity('Three') // error
+//	quantity('Mi') // error
+//
+// isQuantity
+//
+// Returns true if a string is a valid Quantity. isQuantity returns true if and
+// only if quantity does not result in error.
+//
+//	isQuantity( <string>) <bool>
+//
+// Examples:
+//
+//	isQuantity('1.3G') // returns true
+//	isQuantity('1.3Gi') // returns true
+//	isQuantity('1,3G') // returns false
+//	isQuantity('10000k') // returns true
+//	isQuantity('200K') // returns false
+//	isQuantity('Three') // returns false
+//	isQuantity('Mi') // returns false
+//
+// Conversion to Scalars:
+//
+//   - isInteger: returns true if and only if asInteger is safe to call without an error
+//
+//   - asInteger: returns a representation of the current value as an int64 if
+//     possible or results in an error if conversion would result in overflow
+//	   or loss of precision.
+//
+//   - asApproximateFloat: returns a float64 representation of the quantity which may
+//     lose precision. If the value of the quantity is outside the range of a float64
+//     +Inf/-Inf will be returned.
+//
+//     <Quantity>.isInteger() <bool>
+//     <Quantity>.asInteger() <int>
+//     <Quantity>.asApproximateFloat() <float>
+//
+// Examples:
+//
+// quantity("50000000G").isInteger() // returns true
+// quantity("50k").isInteger() // returns true
+// quantity("9999999999999999999999999999999999999G").asInteger() // error: cannot convert value to integer
+// quantity("9999999999999999999999999999999999999G").isInteger() // returns false
+// quantity("50k").asInteger() == 50000 // returns true
+// quantity("50k").sub(20000).asApproximateFloat() == 30000 // returns true
+//
+// Arithmetic
+//
+//   - sign: Returns `1` if the quantity is positive, `-1` if it is negative. `0` if it is zero
+//
+//   - add: Returns sum of two quantities or a quantity and an integer
+//
+//   - sub: Returns difference between two quantities or a quantity and an integer
+//
+//     <Quantity>.sign() <int>
+//     <Quantity>.add(<quantity>) <quantity>
+//     <Quantity>.add(<integer>) <quantity>
+//     <Quantity>.sub(<quantity>) <quantity>
+//     <Quantity>.sub(<integer>) <quantity>
+//
+// Examples:
+//
+// quantity("50k").add("20k") == quantity("70k") // returns true
+// quantity("50k").add(20) == quantity("50020") // returns true
+// quantity("50k").sub("20k") == quantity("30k") // returns true
+// quantity("50k").sub(20000) == quantity("30k") // returns true
+// quantity("50k").add(20).sub(quantity("100k")).sub(-50000) == quantity("20") // returns true
+//
+// Comparisons
+//
+//   - isGreaterThan: Returns true if and only if the receiver is greater than the operand
+//
+//   - isLessThan: Returns true if and only if the receiver is less than the operand
+//
+//   - compareTo: Compares receiver to operand and returns 0 if they are equal, 1 if the receiver is greater, or -1 if the receiver is less than the operand
+//
+//
+//     <Quantity>.isLessThan(<quantity>) <bool>
+//     <Quantity>.isGreaterThan(<quantity>) <bool>
+//     <Quantity>.compareTo(<quantity>) <int>
+//
+// Examples:
+//
+// quantity("200M").compareTo(quantity("0.2G")) // returns 0
+// quantity("50M").compareTo(quantity("50Mi")) // returns -1
+// quantity("50Mi").compareTo(quantity("50M")) // returns 1
+// quantity("150Mi").isGreaterThan(quantity("100Mi")) // returns true
+// quantity("50Mi").isGreaterThan(quantity("100Mi")) // returns false
+// quantity("50M").isLessThan(quantity("100M")) // returns true
+// quantity("100M").isLessThan(quantity("50M")) // returns false
+
+func Quantity() cel.EnvOption {
+	return cel.Lib(quantityLib)
+}
+
+var quantityLib = &quantity{}
+
+type quantity struct{}
+
+var quantityLibraryDecls = map[string][]cel.FunctionOpt{
+	"quantity": {
+		cel.Overload("string_to_quantity", []*cel.Type{cel.StringType}, apiservercel.QuantityType, cel.UnaryBinding((stringToQuantity))),
+	},
+	"isQuantity": {
+		cel.Overload("is_quantity_string", []*cel.Type{cel.StringType}, cel.BoolType, cel.UnaryBinding(isQuantity)),
+	},
+	"sign": {
+		cel.Overload("quantity_sign", []*cel.Type{apiservercel.QuantityType}, cel.IntType, cel.UnaryBinding(quantityGetSign)),
+	},
+	"isGreaterThan": {
+		cel.MemberOverload("quantity_is_greater_than", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.BoolType, cel.BinaryBinding(quantityIsGreaterThan)),
+	},
+	"isLessThan": {
+		cel.MemberOverload("quantity_is_less_than", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.BoolType, cel.BinaryBinding(quantityIsLessThan)),
+	},
+	"compareTo": {
+		cel.MemberOverload("quantity_compare_to", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, cel.IntType, cel.BinaryBinding(quantityCompareTo)),
+	},
+	"asApproximateFloat": {
+		cel.MemberOverload("quantity_get_float", []*cel.Type{apiservercel.QuantityType}, cel.DoubleType, cel.UnaryBinding(quantityGetApproximateFloat)),
+	},
+	"asInteger": {
+		cel.MemberOverload("quantity_get_int", []*cel.Type{apiservercel.QuantityType}, cel.IntType, cel.UnaryBinding(quantityGetValue)),
+	},
+	"isInteger": {
+		cel.MemberOverload("quantity_is_integer", []*cel.Type{apiservercel.QuantityType}, cel.BoolType, cel.UnaryBinding(quantityCanValue)),
+	},
+	"add": {
+		cel.MemberOverload("quantity_add", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, apiservercel.QuantityType, cel.BinaryBinding(quantityAdd)),
+		cel.MemberOverload("quantity_add_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, cel.BinaryBinding(quantityAddInt)),
+	},
+	"sub": {
+		cel.MemberOverload("quantity_sub", []*cel.Type{apiservercel.QuantityType, apiservercel.QuantityType}, apiservercel.QuantityType, cel.BinaryBinding(quantitySub)),
+		cel.MemberOverload("quantity_sub_int", []*cel.Type{apiservercel.QuantityType, cel.IntType}, apiservercel.QuantityType, cel.BinaryBinding(quantitySubInt)),
+	},
+}
+
+func (*quantity) CompileOptions() []cel.EnvOption {
+	options := make([]cel.EnvOption, 0, len(quantityLibraryDecls))
+	for name, overloads := range quantityLibraryDecls {
+		options = append(options, cel.Function(name, overloads...))
+	}
+	return options
+}
+
+func (*quantity) ProgramOptions() []cel.ProgramOption {
+	return []cel.ProgramOption{}
+}
+
+func isQuantity(arg ref.Val) ref.Val {
+	str, ok := arg.Value().(string)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	_, err := resource.ParseQuantity(str)
+	if err != nil {
+		return types.Bool(false)
+	}
+
+	return types.Bool(true)
+}
+
+func stringToQuantity(arg ref.Val) ref.Val {
+	str, ok := arg.Value().(string)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q, err := resource.ParseQuantity(str)
+	if err != nil {
+		return types.WrapErr(err)
+	}
+
+	return apiservercel.Quantity{Quantity: &q}
+}
+
+func quantityGetApproximateFloat(arg ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+	return types.Double(q.AsApproximateFloat64())
+}
+
+func quantityCanValue(arg ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+	_, success := q.AsInt64()
+	return types.Bool(success)
+}
+
+func quantityGetValue(arg ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+	v, success := q.AsInt64()
+	if !success {
+		return types.WrapErr(errors.New("cannot convert value to integer"))
+	}
+	return types.Int(v)
+}
+
+func quantityGetSign(arg ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+	return types.Int(q.Sign())
+}
+
+func quantityIsGreaterThan(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	return types.Bool(q.Cmp(*q2) == 1)
+}
+
+func quantityIsLessThan(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	return types.Bool(q.Cmp(*q2) == -1)
+}
+
+func quantityCompareTo(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	return types.Int(q.Cmp(*q2))
+}
+
+func quantityAdd(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	copy := *q
+	copy.Add(*q2)
+	return &apiservercel.Quantity{
+		Quantity: &copy,
+	}
+}
+
+func quantityAddInt(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(int64)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2Converted := *resource.NewQuantity(q2, resource.DecimalExponent)
+
+	copy := *q
+	copy.Add(q2Converted)
+	return &apiservercel.Quantity{
+		Quantity: &copy,
+	}
+}
+
+func quantitySub(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	copy := *q
+	copy.Sub(*q2)
+	return &apiservercel.Quantity{
+		Quantity: &copy,
+	}
+}
+
+func quantitySubInt(arg ref.Val, other ref.Val) ref.Val {
+	q, ok := arg.Value().(*resource.Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2, ok := other.Value().(int64)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(arg)
+	}
+
+	q2Converted := *resource.NewQuantity(q2, resource.DecimalExponent)
+
+	copy := *q
+	copy.Sub(q2Converted)
+	return &apiservercel.Quantity{
+		Quantity: &copy,
+	}
+}

vendor/k8s.io/apiserver/pkg/cel/library/regex.go

@@ -77,7 +77,9 @@ func (*regex) CompileOptions() []cel.EnvOption {
 }
 
 func (*regex) ProgramOptions() []cel.ProgramOption {
-	return []cel.ProgramOption{}
+	return []cel.ProgramOption{
+		cel.OptimizeRegex(FindRegexOptimization, FindAllRegexOptimization),
+	}
 }
 
 func find(strVal ref.Val, regexVal ref.Val) ref.Val {

vendor/k8s.io/apiserver/pkg/cel/library/test.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package library
+
+import (
+	"math"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+)
+
+// Test provides a test() function that returns true.
+func Test(options ...TestOption) cel.EnvOption {
+	t := &testLib{version: math.MaxUint32}
+	for _, o := range options {
+		t = o(t)
+	}
+	return cel.Lib(t)
+}
+
+type testLib struct {
+	version uint32
+}
+
+type TestOption func(*testLib) *testLib
+
+func TestVersion(version uint32) func(lib *testLib) *testLib {
+	return func(sl *testLib) *testLib {
+		sl.version = version
+		return sl
+	}
+}
+
+func (t *testLib) CompileOptions() []cel.EnvOption {
+	var options []cel.EnvOption
+
+	if t.version == 0 {
+		options = append(options, cel.Function("test",
+			cel.Overload("test", []*cel.Type{}, cel.BoolType,
+				cel.FunctionBinding(func(args ...ref.Val) ref.Val {
+					return types.True
+				}))))
+	}
+
+	if t.version >= 1 {
+		options = append(options, cel.Function("test",
+			cel.Overload("test", []*cel.Type{}, cel.BoolType,
+				cel.FunctionBinding(func(args ...ref.Val) ref.Val {
+					// Return false here so tests can observe which version of the function is registered
+					// Actual function libraries must not break backward compatibility
+					return types.False
+				}))))
+		options = append(options, cel.Function("testV1",
+			cel.Overload("testV1", []*cel.Type{}, cel.BoolType,
+				cel.FunctionBinding(func(args ...ref.Val) ref.Val {
+					return types.True
+				}))))
+	}
+	return options
+}
+
+func (*testLib) ProgramOptions() []cel.ProgramOption {
+	return []cel.ProgramOption{}
+}

vendor/k8s.io/apiserver/pkg/cel/quantity.go

@@ -0,0 +1,76 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker/decls"
+	"github.com/google/cel-go/common/types"
+	"github.com/google/cel-go/common/types/ref"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+var (
+	QuantityObject    = decls.NewObjectType("kubernetes.Quantity")
+	quantityTypeValue = types.NewTypeValue("kubernetes.Quantity")
+	QuantityType      = cel.ObjectType("kubernetes.Quantity")
+)
+
+// Quantity provdes a CEL representation of a resource.Quantity
+type Quantity struct {
+	*resource.Quantity
+}
+
+func (d Quantity) ConvertToNative(typeDesc reflect.Type) (interface{}, error) {
+	if reflect.TypeOf(d.Quantity).AssignableTo(typeDesc) {
+		return d.Quantity, nil
+	}
+	if reflect.TypeOf("").AssignableTo(typeDesc) {
+		return d.Quantity.String(), nil
+	}
+	return nil, fmt.Errorf("type conversion error from 'Quantity' to '%v'", typeDesc)
+}
+
+func (d Quantity) ConvertToType(typeVal ref.Type) ref.Val {
+	switch typeVal {
+	case typeValue:
+		return d
+	case types.TypeType:
+		return quantityTypeValue
+	default:
+		return types.NewErr("type conversion error from '%s' to '%s'", quantityTypeValue, typeVal)
+	}
+}
+
+func (d Quantity) Equal(other ref.Val) ref.Val {
+	otherDur, ok := other.(Quantity)
+	if !ok {
+		return types.MaybeNoSuchOverloadErr(other)
+	}
+	return types.Bool(d.Quantity.Equal(*otherDur.Quantity))
+}
+
+func (d Quantity) Type() ref.Type {
+	return quantityTypeValue
+}
+
+func (d Quantity) Value() interface{} {
+	return d.Quantity
+}

vendor/k8s.io/apiserver/pkg/cel/registry.go

@@ -1,79 +0,0 @@
-/*
-Copyright 2022 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cel
-
-import (
-	"sync"
-
-	"github.com/google/cel-go/cel"
-)
-
-// Resolver declares methods to find policy templates and related configuration objects.
-type Resolver interface {
-	// FindType returns a DeclType instance corresponding to the given fully-qualified name, if
-	// present.
-	FindType(name string) (*DeclType, bool)
-}
-
-// NewRegistry create a registry for keeping track of environments and types
-// from a base cel.Env expression environment.
-func NewRegistry(stdExprEnv *cel.Env) *Registry {
-	return &Registry{
-		exprEnvs: map[string]*cel.Env{"": stdExprEnv},
-		types: map[string]*DeclType{
-			BoolType.TypeName():      BoolType,
-			BytesType.TypeName():     BytesType,
-			DoubleType.TypeName():    DoubleType,
-			DurationType.TypeName():  DurationType,
-			IntType.TypeName():       IntType,
-			NullType.TypeName():      NullType,
-			StringType.TypeName():    StringType,
-			TimestampType.TypeName(): TimestampType,
-			UintType.TypeName():      UintType,
-			ListType.TypeName():      ListType,
-			MapType.TypeName():       MapType,
-		},
-	}
-}
-
-// Registry defines a repository of environment, schema, template, and type definitions.
-//
-// Registry instances are concurrency-safe.
-type Registry struct {
-	rwMux    sync.RWMutex
-	exprEnvs map[string]*cel.Env
-	types    map[string]*DeclType
-}
-
-// FindType implements the Resolver interface method.
-func (r *Registry) FindType(name string) (*DeclType, bool) {
-	r.rwMux.RLock()
-	defer r.rwMux.RUnlock()
-	typ, found := r.types[name]
-	if found {
-		return typ, true
-	}
-	return typ, found
-}
-
-// SetType registers a DeclType descriptor by its fully qualified name.
-func (r *Registry) SetType(name string, declType *DeclType) error {
-	r.rwMux.Lock()
-	defer r.rwMux.Unlock()
-	r.types[name] = declType
-	return nil
-}

vendor/k8s.io/apiserver/pkg/cel/types.go

@@ -319,44 +319,53 @@ func (f *DeclField) EnumValues() []ref.Val {
 	return ev
 }
 
-// NewRuleTypes returns an Open API Schema-based type-system which is CEL compatible.
-func NewRuleTypes(kind string,
-	declType *DeclType,
-	res Resolver) (*RuleTypes, error) {
+func allTypesForDecl(declTypes []*DeclType) map[string]*DeclType {
+	if declTypes == nil {
+		return nil
+	}
+	allTypes := map[string]*DeclType{}
+	for _, declType := range declTypes {
+		for k, t := range FieldTypeMap(declType.TypeName(), declType) {
+			allTypes[k] = t
+		}
+	}
+
+	return allTypes
+}
+
+// NewDeclTypeProvider returns an Open API Schema-based type-system which is CEL compatible.
+func NewDeclTypeProvider(rootTypes ...*DeclType) *DeclTypeProvider {
 	// Note, if the schema indicates that it's actually based on another proto
 	// then prefer the proto definition. For expressions in the proto, a new field
 	// annotation will be needed to indicate the expected environment and type of
 	// the expression.
-	schemaTypes, err := newSchemaTypeProvider(kind, declType)
-	if err != nil {
-		return nil, err
+	allTypes := allTypesForDecl(rootTypes)
+	return &DeclTypeProvider{
+		registeredTypes: allTypes,
 	}
-	if schemaTypes == nil {
-		return nil, nil
-	}
-	return &RuleTypes{
-		ruleSchemaDeclTypes: schemaTypes,
-		resolver:            res,
-	}, nil
 }
 
-// RuleTypes extends the CEL ref.TypeProvider interface and provides an Open API Schema-based
+// DeclTypeProvider extends the CEL ref.TypeProvider interface and provides an Open API Schema-based
 // type-system.
-type RuleTypes struct {
-	ref.TypeProvider
-	ruleSchemaDeclTypes *schemaTypeProvider
-	typeAdapter         ref.TypeAdapter
-	resolver            Resolver
+type DeclTypeProvider struct {
+	registeredTypes map[string]*DeclType
+	typeProvider    ref.TypeProvider
+	typeAdapter     ref.TypeAdapter
+}
+
+func (rt *DeclTypeProvider) EnumValue(enumName string) ref.Val {
+	return rt.typeProvider.EnumValue(enumName)
+}
+
+func (rt *DeclTypeProvider) FindIdent(identName string) (ref.Val, bool) {
+	return rt.typeProvider.FindIdent(identName)
 }
 
 // EnvOptions returns a set of cel.EnvOption values which includes the declaration set
 // as well as a custom ref.TypeProvider.
 //
-// Note, the standard declaration set includes 'rule' which is defined as the top-level rule-schema
-// type if one is configured.
-//
-// If the RuleTypes value is nil, an empty []cel.EnvOption set is returned.
-func (rt *RuleTypes) EnvOptions(tp ref.TypeProvider) ([]cel.EnvOption, error) {
+// If the DeclTypeProvider value is nil, an empty []cel.EnvOption set is returned.
+func (rt *DeclTypeProvider) EnvOptions(tp ref.TypeProvider) ([]cel.EnvOption, error) {
 	if rt == nil {
 		return []cel.EnvOption{}, nil
 	}
@@ -367,13 +376,12 @@ func (rt *RuleTypes) EnvOptions(tp ref.TypeProvider) ([]cel.EnvOption, error) {
 	return []cel.EnvOption{
 		cel.CustomTypeProvider(rtWithTypes),
 		cel.CustomTypeAdapter(rtWithTypes),
-		cel.Variable("rule", rt.ruleSchemaDeclTypes.root.CelType()),
 	}, nil
 }
 
-// WithTypeProvider returns a new RuleTypes that sets the given TypeProvider
-// If the original RuleTypes is nil, the returned RuleTypes is still nil.
-func (rt *RuleTypes) WithTypeProvider(tp ref.TypeProvider) (*RuleTypes, error) {
+// WithTypeProvider returns a new DeclTypeProvider that sets the given TypeProvider
+// If the original DeclTypeProvider is nil, the returned DeclTypeProvider is still nil.
+func (rt *DeclTypeProvider) WithTypeProvider(tp ref.TypeProvider) (*DeclTypeProvider, error) {
 	if rt == nil {
 		return nil, nil
 	}
@@ -382,13 +390,12 @@ func (rt *RuleTypes) WithTypeProvider(tp ref.TypeProvider) (*RuleTypes, error) {
 	if ok {
 		ta = tpa
 	}
-	rtWithTypes := &RuleTypes{
-		TypeProvider:        tp,
-		typeAdapter:         ta,
-		ruleSchemaDeclTypes: rt.ruleSchemaDeclTypes,
-		resolver:            rt.resolver,
+	rtWithTypes := &DeclTypeProvider{
+		typeProvider:    tp,
+		typeAdapter:     ta,
+		registeredTypes: rt.registeredTypes,
 	}
-	for name, declType := range rt.ruleSchemaDeclTypes.types {
+	for name, declType := range rt.registeredTypes {
 		tpType, found := tp.FindType(name)
 		expT, err := declType.ExprType()
 		if err != nil {
@@ -396,7 +403,7 @@ func (rt *RuleTypes) WithTypeProvider(tp ref.TypeProvider) (*RuleTypes, error) {
 		}
 		if found && !proto.Equal(tpType, expT) {
 			return nil, fmt.Errorf(
-				"type %s definition differs between CEL environment and rule", name)
+				"type %s definition differs between CEL environment and type provider", name)
 		}
 	}
 	return rtWithTypes, nil
@@ -409,7 +416,7 @@ func (rt *RuleTypes) WithTypeProvider(tp ref.TypeProvider) (*RuleTypes, error) {
 //
 // Note, when the type name is based on the Open API Schema, the name will reflect the object path
 // where the type definition appears.
-func (rt *RuleTypes) FindType(typeName string) (*exprpb.Type, bool) {
+func (rt *DeclTypeProvider) FindType(typeName string) (*exprpb.Type, bool) {
 	if rt == nil {
 		return nil, false
 	}
@@ -421,11 +428,11 @@ func (rt *RuleTypes) FindType(typeName string) (*exprpb.Type, bool) {
 		}
 		return expT, found
 	}
-	return rt.TypeProvider.FindType(typeName)
+	return rt.typeProvider.FindType(typeName)
 }
 
 // FindDeclType returns the CPT type description which can be mapped to a CEL type.
-func (rt *RuleTypes) FindDeclType(typeName string) (*DeclType, bool) {
+func (rt *DeclTypeProvider) FindDeclType(typeName string) (*DeclType, bool) {
 	if rt == nil {
 		return nil, false
 	}
@@ -438,10 +445,10 @@ func (rt *RuleTypes) FindDeclType(typeName string) (*DeclType, bool) {
 // If, in the future an object instance rather than a type name were provided, the field
 // resolution might more accurately reflect the expected type model. However, in this case
 // concessions were made to align with the existing CEL interfaces.
-func (rt *RuleTypes) FindFieldType(typeName, fieldName string) (*ref.FieldType, bool) {
+func (rt *DeclTypeProvider) FindFieldType(typeName, fieldName string) (*ref.FieldType, bool) {
 	st, found := rt.findDeclType(typeName)
 	if !found {
-		return rt.TypeProvider.FindFieldType(typeName, fieldName)
+		return rt.typeProvider.FindFieldType(typeName, fieldName)
 	}
 
 	f, found := st.Fields[fieldName]
@@ -471,48 +478,63 @@ func (rt *RuleTypes) FindFieldType(typeName, fieldName string) (*ref.FieldType,
 
 // NativeToValue is an implementation of the ref.TypeAdapater interface which supports conversion
 // of rule values to CEL ref.Val instances.
-func (rt *RuleTypes) NativeToValue(val interface{}) ref.Val {
+func (rt *DeclTypeProvider) NativeToValue(val interface{}) ref.Val {
 	return rt.typeAdapter.NativeToValue(val)
 }
 
-// TypeNames returns the list of type names declared within the RuleTypes object.
-func (rt *RuleTypes) TypeNames() []string {
-	typeNames := make([]string, len(rt.ruleSchemaDeclTypes.types))
+func (rt *DeclTypeProvider) NewValue(typeName string, fields map[string]ref.Val) ref.Val {
+	// TODO: implement for OpenAPI types to enable CEL object instantiation, which is needed
+	// for mutating admission.
+	return rt.typeProvider.NewValue(typeName, fields)
+}
+
+// TypeNames returns the list of type names declared within the DeclTypeProvider object.
+func (rt *DeclTypeProvider) TypeNames() []string {
+	typeNames := make([]string, len(rt.registeredTypes))
 	i := 0
-	for name := range rt.ruleSchemaDeclTypes.types {
+	for name := range rt.registeredTypes {
 		typeNames[i] = name
 		i++
 	}
 	return typeNames
 }
 
-func (rt *RuleTypes) findDeclType(typeName string) (*DeclType, bool) {
-	declType, found := rt.ruleSchemaDeclTypes.types[typeName]
+func (rt *DeclTypeProvider) findDeclType(typeName string) (*DeclType, bool) {
+	declType, found := rt.registeredTypes[typeName]
 	if found {
 		return declType, true
 	}
-	declType, found = rt.resolver.FindType(typeName)
-	if found {
-		return declType, true
-	}
-	return nil, false
+	declType = findScalar(typeName)
+	return declType, declType != nil
 }
 
-func newSchemaTypeProvider(kind string, declType *DeclType) (*schemaTypeProvider, error) {
-	if declType == nil {
-		return nil, nil
+func findScalar(typename string) *DeclType {
+	switch typename {
+	case BoolType.TypeName():
+		return BoolType
+	case BytesType.TypeName():
+		return BytesType
+	case DoubleType.TypeName():
+		return DoubleType
+	case DurationType.TypeName():
+		return DurationType
+	case IntType.TypeName():
+		return IntType
+	case NullType.TypeName():
+		return NullType
+	case StringType.TypeName():
+		return StringType
+	case TimestampType.TypeName():
+		return TimestampType
+	case UintType.TypeName():
+		return UintType
+	case ListType.TypeName():
+		return ListType
+	case MapType.TypeName():
+		return MapType
+	default:
+		return nil
 	}
-	root := declType.MaybeAssignTypeName(kind)
-	types := FieldTypeMap(kind, root)
-	return &schemaTypeProvider{
-		root:  root,
-		types: types,
-	}, nil
-}
-
-type schemaTypeProvider struct {
-	root  *DeclType
-	types map[string]*DeclType
 }
 
 var (

vendor/k8s.io/apiserver/pkg/endpoints/filters/audit.go

@@ -51,11 +51,11 @@ func WithAudit(handler http.Handler, sink audit.Sink, policy audit.PolicyRuleEva
 			return
 		}
 
-		if ac == nil || ac.Event == nil {
+		if !ac.Enabled() {
 			handler.ServeHTTP(w, req)
 			return
 		}
-		ev := ac.Event
+		ev := &ac.Event
 
 		ctx := req.Context()
 		omitStages := ac.RequestAuditConfig.OmitStages
@@ -124,7 +124,7 @@ func evaluatePolicyAndCreateAuditEvent(req *http.Request, policy audit.PolicyRul
 	ctx := req.Context()
 	ac := audit.AuditContextFrom(ctx)
 	if ac == nil {
-		// Auditing not enabled.
+		// Auditing not configured.
 		return nil, nil
 	}
 
@@ -145,12 +145,7 @@ func evaluatePolicyAndCreateAuditEvent(req *http.Request, policy audit.PolicyRul
 	if !ok {
 		requestReceivedTimestamp = time.Now()
 	}
-	ev, err := audit.NewEventFromRequest(req, requestReceivedTimestamp, rac.Level, attribs)
-	if err != nil {
-		return nil, fmt.Errorf("failed to complete audit event from request: %v", err)
-	}
-
-	ac.Event = ev
+	audit.LogRequestMetadata(ctx, req, requestReceivedTimestamp, rac.Level, attribs)
 
 	return ac, nil
 }

vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go

@@ -34,17 +34,17 @@ import (
 	"k8s.io/klog/v2"
 )
 
-type recordMetrics func(context.Context, *authenticator.Response, bool, error, authenticator.Audiences, time.Time, time.Time)
+type authenticationRecordMetricsFunc func(context.Context, *authenticator.Response, bool, error, authenticator.Audiences, time.Time, time.Time)
 
 // WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then
 // stores any such user found onto the provided context for the request. If authentication fails or returns an error
 // the failed handler is used. On success, "Authorization" header is removed from the request and handler
 // is invoked to serve the request.
 func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, requestHeaderConfig *authenticatorfactory.RequestHeaderConfig) http.Handler {
-	return withAuthentication(handler, auth, failed, apiAuds, requestHeaderConfig, recordAuthMetrics)
+	return withAuthentication(handler, auth, failed, apiAuds, requestHeaderConfig, recordAuthenticationMetrics)
 }
 
-func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, requestHeaderConfig *authenticatorfactory.RequestHeaderConfig, metrics recordMetrics) http.Handler {
+func withAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler, apiAuds authenticator.Audiences, requestHeaderConfig *authenticatorfactory.RequestHeaderConfig, metrics authenticationRecordMetricsFunc) http.Handler {
 	if auth == nil {
 		klog.Warning("Authentication is disabled")
 		return handler

vendor/k8s.io/apiserver/pkg/endpoints/filters/authn_audit.go

@@ -43,11 +43,11 @@ func WithFailedAuthenticationAudit(failedHandler http.Handler, sink audit.Sink,
 			return
 		}
 
-		if ac == nil || ac.Event == nil {
+		if !ac.Enabled() {
 			failedHandler.ServeHTTP(w, req)
 			return
 		}
-		ev := ac.Event
+		ev := &ac.Event
 
 		ev.ResponseStatus = &metav1.Status{}
 		ev.ResponseStatus.Message = getAuthMethods(req)

vendor/k8s.io/apiserver/pkg/endpoints/filters/authorization.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"time"
 
 	"k8s.io/klog/v2"
 
@@ -41,14 +42,21 @@ const (
 	reasonError    = "internal error"
 )
 
-// WithAuthorizationCheck passes all authorized requests on to handler, and returns a forbidden error otherwise.
-func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {
+type recordAuthorizationMetricsFunc func(ctx context.Context, authorized authorizer.Decision, err error, authStart time.Time, authFinish time.Time)
+
+// WithAuthorization passes all authorized requests on to handler, and returns a forbidden error otherwise.
+func WithAuthorization(hhandler http.Handler, auth authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {
+	return withAuthorization(hhandler, auth, s, recordAuthorizationMetrics)
+}
+
+func withAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer, metrics recordAuthorizationMetricsFunc) http.Handler {
 	if a == nil {
 		klog.Warning("Authorization is disabled")
 		return handler
 	}
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		ctx := req.Context()
+		authorizationStart := time.Now()
 
 		attributes, err := GetAuthorizerAttributes(ctx)
 		if err != nil {
@@ -56,6 +64,12 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.
 			return
 		}
 		authorized, reason, err := a.Authorize(ctx, attributes)
+
+		authorizationFinish := time.Now()
+		defer func() {
+			metrics(ctx, authorized, err, authorizationStart, authorizationFinish)
+		}()
+
 		// an authorizer like RBAC could encounter evaluation errors and still allow the request, so authorizer decision is checked before error here.
 		if authorized == authorizer.DecisionAllow {
 			audit.AddAuditAnnotations(ctx,

vendor/k8s.io/apiserver/pkg/endpoints/filters/metrics.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	"k8s.io/apiserver/pkg/authorization/authorizer"
+
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
@@ -38,6 +40,10 @@ const (
 	successLabel = "success"
 	failureLabel = "failure"
 	errorLabel   = "error"
+
+	allowedLabel   = "allowed"
+	deniedLabel    = "denied"
+	noOpinionLabel = "no-opinion"
 )
 
 var (
@@ -68,15 +74,54 @@ var (
 		},
 		[]string{"result"},
 	)
+
+	authorizationAttemptsCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "authorization_attempts_total",
+			Help:           "Counter of authorization attempts broken down by result. It can be either 'allowed', 'denied', 'no-opinion' or 'error'.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"result"},
+	)
+
+	authorizationLatency = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Name:           "authorization_duration_seconds",
+			Help:           "Authorization duration in seconds broken out by result.",
+			Buckets:        metrics.ExponentialBuckets(0.001, 2, 15),
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"result"},
+	)
 )
 
 func init() {
 	legacyregistry.MustRegister(authenticatedUserCounter)
 	legacyregistry.MustRegister(authenticatedAttemptsCounter)
 	legacyregistry.MustRegister(authenticationLatency)
+	legacyregistry.MustRegister(authorizationAttemptsCounter)
+	legacyregistry.MustRegister(authorizationLatency)
 }
 
-func recordAuthMetrics(ctx context.Context, resp *authenticator.Response, ok bool, err error, apiAudiences authenticator.Audiences, authStart time.Time, authFinish time.Time) {
+func recordAuthorizationMetrics(ctx context.Context, authorized authorizer.Decision, err error, authStart time.Time, authFinish time.Time) {
+	var resultLabel string
+
+	switch {
+	case authorized == authorizer.DecisionAllow:
+		resultLabel = allowedLabel
+	case err != nil:
+		resultLabel = errorLabel
+	case authorized == authorizer.DecisionDeny:
+		resultLabel = deniedLabel
+	case authorized == authorizer.DecisionNoOpinion:
+		resultLabel = noOpinionLabel
+	}
+
+	authorizationAttemptsCounter.WithContext(ctx).WithLabelValues(resultLabel).Inc()
+	authorizationLatency.WithContext(ctx).WithLabelValues(resultLabel).Observe(authFinish.Sub(authStart).Seconds())
+}
+
+func recordAuthenticationMetrics(ctx context.Context, resp *authenticator.Response, ok bool, err error, apiAudiences authenticator.Audiences, authStart time.Time, authFinish time.Time) {
 	var resultLabel string
 
 	switch {

vendor/k8s.io/apiserver/pkg/endpoints/filters/request_deadline.go

@@ -115,11 +115,11 @@ func withFailedRequestAudit(failedHandler http.Handler, statusErr *apierrors.Sta
 			return
 		}
 
-		if ac == nil || ac.Event == nil {
+		if !ac.Enabled() {
 			failedHandler.ServeHTTP(w, req)
 			return
 		}
-		ev := ac.Event
+		ev := &ac.Event
 
 		ev.ResponseStatus = &metav1.Status{}
 		ev.Stage = auditinternal.StageResponseStarted

vendor/k8s.io/apiserver/pkg/endpoints/groupversion.go

@@ -56,6 +56,11 @@ type APIGroupVersion struct {
 	// GroupVersion is the external group version
 	GroupVersion schema.GroupVersion
 
+	// AllServedVersionsByResource is indexed by resource and maps to a list of versions that resource exists in.
+	// This was created so that StorageVersion for APIs can include a list of all version that are served for each
+	// GroupResource tuple.
+	AllServedVersionsByResource map[string][]string
+
 	// OptionsExternalVersion controls the Kubernetes APIVersion used for common objects in the apiserver
 	// schema like api.Status, api.DeleteOptions, and metav1.ListOptions. Other implementors may
 	// define a version "v1beta1" but want to use the Kubernetes "v1" internal objects. If

vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go

@@ -191,14 +191,13 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 		// Dedup owner references before updating managed fields
 		dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
 		result, err := finisher.FinishRequest(ctx, func() (runtime.Object, error) {
-			if scope.FieldManager != nil {
-				liveObj, err := scope.Creater.New(scope.Kind)
-				if err != nil {
-					return nil, fmt.Errorf("failed to create new object (Create for %v): %v", scope.Kind, err)
-				}
-				obj = scope.FieldManager.UpdateNoErrors(liveObj, obj, managerOrUserAgent(options.FieldManager, req.UserAgent()))
-				admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
+			liveObj, err := scope.Creater.New(scope.Kind)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create new object (Create for %v): %v", scope.Kind, err)
 			}
+			obj = scope.FieldManager.UpdateNoErrors(liveObj, obj, managerOrUserAgent(options.FieldManager, req.UserAgent()))
+			admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
+
 			if mutatingAdmission, ok := admit.(admission.MutationInterface); ok && mutatingAdmission.Handles(admission.Create) {
 				if err := mutatingAdmission.Admit(ctx, admissionAttributes, scope); err != nil {
 					return nil, err

vendor/k8s.io/apiserver/pkg/endpoints/handlers/patch.go

@@ -177,9 +177,8 @@ func PatchResource(r rest.Patcher, scope *RequestScope, admit admission.Interfac
 			userInfo,
 		)
 
-		if scope.FieldManager != nil {
-			admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
-		}
+		admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
+
 		mutatingAdmission, _ := admit.(admission.MutationInterface)
 		createAuthorizerAttributes := authorizer.AttributesRecord{
 			User:            userInfo,
@@ -345,9 +344,12 @@ func (p *jsonPatcher) applyPatchToCurrentObject(requestContext context.Context,
 		}
 	}
 
-	if p.fieldManager != nil {
-		objToUpdate = p.fieldManager.UpdateNoErrors(currentObject, objToUpdate, managerOrUserAgent(p.options.FieldManager, p.userAgent))
+	if p.options == nil {
+		// Provide a more informative error for the crash that would
+		// happen on the next line
+		panic("PatchOptions required but not provided")
 	}
+	objToUpdate = p.fieldManager.UpdateNoErrors(currentObject, objToUpdate, managerOrUserAgent(p.options.FieldManager, p.userAgent))
 	return objToUpdate, nil
 }
 
@@ -441,9 +443,7 @@ func (p *smpPatcher) applyPatchToCurrentObject(requestContext context.Context, c
 		return nil, err
 	}
 
-	if p.fieldManager != nil {
-		newObj = p.fieldManager.UpdateNoErrors(currentObject, newObj, managerOrUserAgent(p.options.FieldManager, p.userAgent))
-	}
+	newObj = p.fieldManager.UpdateNoErrors(currentObject, newObj, managerOrUserAgent(p.options.FieldManager, p.userAgent))
 	return newObj, nil
 }
 
@@ -654,9 +654,6 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
 	}
 
 	transformers := []rest.TransformFunc{p.applyPatch, p.applyAdmission, dedupOwnerReferencesTransformer}
-	if scope.FieldManager != nil {
-		transformers = append(transformers, fieldmanager.IgnoreManagedFieldsTimestampsTransformer)
-	}
 
 	wasCreated := false
 	p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, transformers...)

vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go

@@ -34,6 +34,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/httpstream/wsstream"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
@@ -42,7 +43,6 @@ import (
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/apiserver/pkg/util/flushwriter"
-	"k8s.io/apiserver/pkg/util/wsstream"
 	"k8s.io/component-base/tracing"
 )
 

vendor/k8s.io/apiserver/pkg/endpoints/handlers/update.go

@@ -156,15 +156,13 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 
 		// allows skipping managedFields update if the resulting object is too big
 		shouldUpdateManagedFields := true
-		if scope.FieldManager != nil {
-			admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
-			transformers = append(transformers, func(_ context.Context, newObj, liveObj runtime.Object) (runtime.Object, error) {
-				if shouldUpdateManagedFields {
-					return scope.FieldManager.UpdateNoErrors(liveObj, newObj, managerOrUserAgent(options.FieldManager, req.UserAgent())), nil
-				}
-				return newObj, nil
-			})
-		}
+		admit = fieldmanager.NewManagedFieldsValidatingAdmissionController(admit)
+		transformers = append(transformers, func(_ context.Context, newObj, liveObj runtime.Object) (runtime.Object, error) {
+			if shouldUpdateManagedFields {
+				return scope.FieldManager.UpdateNoErrors(liveObj, newObj, managerOrUserAgent(options.FieldManager, req.UserAgent())), nil
+			}
+			return newObj, nil
+		})
 
 		if mutatingAdmission, ok := admit.(admission.MutationInterface); ok {
 			transformers = append(transformers, func(ctx context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) {
@@ -189,15 +187,6 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 			})
 		}
 
-		// Ignore changes that only affect managed fields
-		// timestamps. FieldManager can't know about changes
-		// like normalized fields, defaulted fields and other
-		// mutations.
-		// Only makes sense when SSA field manager is being used
-		if scope.FieldManager != nil {
-			transformers = append(transformers, fieldmanager.IgnoreManagedFieldsTimestampsTransformer)
-		}
-
 		createAuthorizerAttributes := authorizer.AttributesRecord{
 			User:            userInfo,
 			ResourceRequest: true,
@@ -237,7 +226,7 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 			result, err := requestFunc()
 			// If the object wasn't committed to storage because it's serialized size was too large,
 			// it is safe to remove managedFields (which can be large) and try again.
-			if isTooLargeError(err) && scope.FieldManager != nil {
+			if isTooLargeError(err) {
 				if accessor, accessorErr := meta.Accessor(obj); accessorErr == nil {
 					accessor.SetManagedFields(nil)
 					shouldUpdateManagedFields = false

vendor/k8s.io/apiserver/pkg/endpoints/handlers/watch.go

@@ -30,12 +30,12 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer/streaming"
+	"k8s.io/apimachinery/pkg/util/httpstream/wsstream"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/apiserver/pkg/util/wsstream"
 )
 
 // nothing will ever be sent down this channel
@@ -219,7 +219,7 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	var unknown runtime.Unknown
 	internalEvent := &metav1.InternalEvent{}
 	outEvent := &metav1.WatchEvent{}
-	buf := &bytes.Buffer{}
+	buf := runtime.NewSpliceBuffer()
 	ch := s.Watching.ResultChan()
 	done := req.Context().Done()
 

vendor/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -127,6 +127,9 @@ func ConvertGroupVersionIntoToDiscovery(list []metav1.APIResource) ([]apidiscove
 			apiResourceList = append(apiResourceList, apidiscoveryv2beta1.APIResourceDiscovery{
 				Resource: split[0],
 				Scope:    scope,
+				// avoid nil panics in v0.26.0-v0.26.3 client-go clients
+				// see https://github.com/kubernetes/kubernetes/issues/118361
+				ResponseKind: &metav1.GroupVersionKind{},
 			})
 			parentidx = len(apiResourceList) - 1
 			parentResources[split[0]] = parentidx
@@ -140,6 +143,9 @@ func ConvertGroupVersionIntoToDiscovery(list []metav1.APIResource) ([]apidiscove
 		subresource := apidiscoveryv2beta1.APISubresourceDiscovery{
 			Subresource: split[1],
 			Verbs:       r.Verbs,
+			// avoid nil panics in v0.26.0-v0.26.3 client-go clients
+			// see https://github.com/kubernetes/kubernetes/issues/118361
+			ResponseKind: &metav1.GroupVersionKind{},
 		}
 		if r.Kind != "" {
 			subresource.ResponseKind = &metav1.GroupVersionKind{
@@ -600,6 +606,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		if a.group.ConvertabilityChecker != nil {
 			decodableVersions = a.group.ConvertabilityChecker.VersionsForGroupKind(fqKindToRegister.GroupKind())
 		}
+
 		resourceInfo = &storageversion.ResourceInfo{
 			GroupResource: schema.GroupResource{
 				Group:    a.group.GroupVersion.Group,
@@ -612,6 +619,8 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			EquivalentResourceMapper: a.group.EquivalentResourceRegistry,
 
 			DirectlyDecodableVersions: decodableVersions,
+
+			ServedVersions: a.group.AllServedVersionsByResource[path],
 		}
 	}
 
@@ -674,28 +683,23 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		reqScope.MetaGroupVersion = *a.group.MetaGroupVersion
 	}
 
-	// Use TypeConverter's nil-ness as a proxy for whether SSA/OpenAPI is enabled
-	// This should be removed in the future and made unconditional
-	// https://github.com/kubernetes/kubernetes/pull/114998
-	if a.group.TypeConverter != nil {
-		var resetFields map[fieldpath.APIVersion]*fieldpath.Set
-		if resetFieldsStrategy, isResetFieldsStrategy := storage.(rest.ResetFieldsStrategy); isResetFieldsStrategy {
-			resetFields = resetFieldsStrategy.GetResetFields()
-		}
+	var resetFields map[fieldpath.APIVersion]*fieldpath.Set
+	if resetFieldsStrategy, isResetFieldsStrategy := storage.(rest.ResetFieldsStrategy); isResetFieldsStrategy {
+		resetFields = resetFieldsStrategy.GetResetFields()
+	}
 
-		reqScope.FieldManager, err = managedfields.NewDefaultFieldManager(
-			a.group.TypeConverter,
-			a.group.UnsafeConvertor,
-			a.group.Defaulter,
-			a.group.Creater,
-			fqKindToRegister,
-			reqScope.HubGroupVersion,
-			subresource,
-			resetFields,
-		)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to create field manager: %v", err)
-		}
+	reqScope.FieldManager, err = managedfields.NewDefaultFieldManager(
+		a.group.TypeConverter,
+		a.group.UnsafeConvertor,
+		a.group.Defaulter,
+		a.group.Creater,
+		fqKindToRegister,
+		reqScope.HubGroupVersion,
+		subresource,
+		resetFields,
+	)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
 
 	for _, action := range actions {
@@ -716,7 +720,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			requestScope = "resource"
 			operationSuffix = operationSuffix + "WithPath"
 		}
-		if strings.Index(action.Path, "/{name}") != -1 || action.Verb == "POST" {
+		if strings.Contains(action.Path, "/{name}") || action.Verb == "POST" {
 			requestScope = "resource"
 		}
 		if action.AllNamespaces {

vendor/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -229,7 +229,7 @@ var (
 			Subsystem:      APIServerComponent,
 			Name:           "request_filter_duration_seconds",
 			Help:           "Request filter latency distribution in seconds, for each filter type",
-			Buckets:        []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
+			Buckets:        []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0, 10.0, 15.0, 30.0},
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{"filter"},

vendor/k8s.io/apiserver/pkg/features/kube_features.go

@@ -37,6 +37,7 @@ const (
 
 	// owner: @ivelichkovich, @tallclair
 	// alpha: v1.27
+	// beta: v1.28
 	// kep: https://kep.k8s.io/3716
 	//
 	// Enables usage of MatchConditions fields to use CEL expressions for matching on admission webhooks
@@ -87,16 +88,6 @@ const (
 	// Add support for distributed tracing in the API Server
 	APIServerTracing featuregate.Feature = "APIServerTracing"
 
-	// owner: @tallclair
-	// alpha: v1.7
-	// beta: v1.8
-	// GA: v1.12
-	//
-	// AdvancedAuditing enables a much more general API auditing pipeline, which includes support for
-	// pluggable output backends and an audit policy specifying how different requests should be
-	// audited.
-	AdvancedAuditing featuregate.Feature = "AdvancedAuditing"
-
 	// owner: @cici37 @jpbetz
 	// kep: http://kep.k8s.io/3488
 	// alpha: v1.26
@@ -112,17 +103,6 @@ const (
 	// Enables expression validation for Custom Resource
 	CustomResourceValidationExpressions featuregate.Feature = "CustomResourceValidationExpressions"
 
-	// owner: @apelisse
-	// alpha: v1.12
-	// beta: v1.13
-	// stable: v1.18
-	//
-	// Allow requests to be processed but not stored, so that
-	// validation, merging, mutation can be tested without
-	// committing.
-	DryRun featuregate.Feature = "DryRun"
-
-	// owner: @wojtek-t
 	// alpha: v1.20
 	// beta: v1.21
 	// GA: v1.24
@@ -130,6 +110,13 @@ const (
 	// Allows for updating watchcache resource version with progress notify events.
 	EfficientWatchResumption featuregate.Feature = "EfficientWatchResumption"
 
+	// owner: @aramase
+	// kep: https://kep.k8s.io/3299
+	// deprecated: v1.28
+	//
+	// Enables KMS v1 API for encryption at rest.
+	KMSv1 featuregate.Feature = "KMSv1"
+
 	// owner: @aramase
 	// kep: https://kep.k8s.io/3299
 	// alpha: v1.25
@@ -138,6 +125,13 @@ const (
 	// Enables KMS v2 API for encryption at rest.
 	KMSv2 featuregate.Feature = "KMSv2"
 
+	// owner: @enj
+	// kep: https://kep.k8s.io/3299
+	// beta: v1.28
+	//
+	// Enables the use of derived encryption keys with KMS v2.
+	KMSv2KDF featuregate.Feature = "KMSv2KDF"
+
 	// owner: @jiahuif
 	// kep: https://kep.k8s.io/2887
 	// alpha: v1.23
@@ -222,6 +216,13 @@ const (
 	//
 	// Allow the API server to stream individual items instead of chunking
 	WatchList featuregate.Feature = "WatchList"
+
+	// owner: @serathius
+	// kep: http://kep.k8s.io/2340
+	// alpha: v1.28
+	//
+	// Allow the API server to serve consistent lists from cache
+	ConsistentListFromCache featuregate.Feature = "ConsistentListFromCache"
 )
 
 func init() {
@@ -235,7 +236,7 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	AggregatedDiscoveryEndpoint: {Default: true, PreRelease: featuregate.Beta},
 
-	AdmissionWebhookMatchConditions: {Default: false, PreRelease: featuregate.Alpha},
+	AdmissionWebhookMatchConditions: {Default: true, PreRelease: featuregate.Beta},
 
 	APIListChunking: {Default: true, PreRelease: featuregate.Beta},
 
@@ -247,18 +248,18 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	APIServerTracing: {Default: true, PreRelease: featuregate.Beta},
 
-	AdvancedAuditing: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.28
-
-	ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Alpha},
+	ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Beta},
 
 	CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.Beta},
 
-	DryRun: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.28
-
 	EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
+	KMSv1: {Default: true, PreRelease: featuregate.Deprecated},
+
 	KMSv2: {Default: true, PreRelease: featuregate.Beta},
 
+	KMSv2KDF: {Default: false, PreRelease: featuregate.Beta}, // default and lock to true in 1.29, remove in 1.31
+
 	OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},
 
 	OpenAPIV3: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29
@@ -280,4 +281,6 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	InPlacePodVerticalScaling: {Default: false, PreRelease: featuregate.Alpha},
 
 	WatchList: {Default: false, PreRelease: featuregate.Alpha},
+
+	ConsistentListFromCache: {Default: false, PreRelease: featuregate.Alpha},
 }

vendor/k8s.io/apiserver/pkg/registry/generic/OWNERS

@@ -2,7 +2,6 @@
 
 reviewers:
   - thockin
-  - lavalamp
   - smarterclayton
   - wojtek-t
   - deads2k

vendor/k8s.io/apiserver/pkg/registry/generic/registry/dryrun.go

@@ -18,7 +18,10 @@ package registry
 
 import (
 	"context"
+	"fmt"
+	"reflect"
 
+	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage"
@@ -72,19 +75,30 @@ func (s *DryRunnableStorage) GuaranteedUpdate(
 	ctx context.Context, key string, destination runtime.Object, ignoreNotFound bool,
 	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, dryRun bool, cachedExistingObject runtime.Object) error {
 	if dryRun {
-		err := s.Storage.Get(ctx, key, storage.GetOptions{IgnoreNotFound: ignoreNotFound}, destination)
+		var current runtime.Object
+		v, err := conversion.EnforcePtr(destination)
+		if err != nil {
+			return fmt.Errorf("unable to convert output object to pointer: %v", err)
+		}
+		if u, ok := v.Addr().Interface().(runtime.Unstructured); ok {
+			current = u.NewEmptyInstance()
+		} else {
+			current = reflect.New(v.Type()).Interface().(runtime.Object)
+		}
+
+		err = s.Storage.Get(ctx, key, storage.GetOptions{IgnoreNotFound: ignoreNotFound}, current)
 		if err != nil {
 			return err
 		}
-		err = preconditions.Check(key, destination)
+		err = preconditions.Check(key, current)
 		if err != nil {
 			return err
 		}
-		rev, err := s.Versioner().ObjectResourceVersion(destination)
+		rev, err := s.Versioner().ObjectResourceVersion(current)
 		if err != nil {
 			return err
 		}
-		updated, _, err := tryUpdate(destination, storage.ResponseMeta{ResourceVersion: rev})
+		updated, _, err := tryUpdate(current, storage.ResponseMeta{ResourceVersion: rev})
 		if err != nil {
 			return err
 		}

vendor/k8s.io/apiserver/pkg/registry/generic/registry/store.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -671,6 +672,15 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 		if err := rest.BeforeUpdate(e.UpdateStrategy, ctx, obj, existing); err != nil {
 			return nil, nil, err
 		}
+
+		// Ignore changes that only affect managed fields timestamps.
+		// FieldManager can't know about changes like normalized fields, defaulted
+		// fields and other mutations.
+		obj, err = fieldmanager.IgnoreManagedFieldsTimestampsTransformer(ctx, obj, existing)
+		if err != nil {
+			return nil, nil, err
+		}
+
 		// at this point we have a fully formed object.  It is time to call the validators that the apiserver
 		// handling chain wants to enforce.
 		if updateValidation != nil {
@@ -1133,6 +1143,11 @@ func (e *Store) DeleteReturnsDeletedObject() bool {
 	return e.ReturnDeletedObject
 }
 
+// deleteCollectionPageSize is the size of the page used when
+// listing objects from storage during DeleteCollection calls.
+// It's a variable to make allow overwriting in tests.
+var deleteCollectionPageSize = int64(10000)
+
 // DeleteCollection removes all items returned by List with a given ListOptions from storage.
 //
 // DeleteCollection is currently NOT atomic. It can happen that only subset of objects
@@ -1145,32 +1160,22 @@ func (e *Store) DeleteCollection(ctx context.Context, deleteValidation rest.Vali
 		listOptions = listOptions.DeepCopy()
 	}
 
-	listObj, err := e.List(ctx, listOptions)
-	if err != nil {
-		return nil, err
-	}
-	items, err := meta.ExtractList(listObj)
-	if err != nil {
-		return nil, err
-	}
-	if len(items) == 0 {
-		// Nothing to delete, return now
-		return listObj, nil
-	}
-	// Spawn a number of goroutines, so that we can issue requests to storage
-	// in parallel to speed up deletion.
-	// It is proportional to the number of items to delete, up to
-	// DeleteCollectionWorkers (it doesn't make much sense to spawn 16
-	// workers to delete 10 items).
+	var items []runtime.Object
+
+	// TODO(wojtek-t): Decide if we don't want to start workers more opportunistically.
 	workersNumber := e.DeleteCollectionWorkers
-	if workersNumber > len(items) {
-		workersNumber = len(items)
-	}
 	if workersNumber < 1 {
 		workersNumber = 1
 	}
 	wg := sync.WaitGroup{}
-	toProcess := make(chan int, 2*workersNumber)
+	// Ensure that chanSize is not too high (to avoid wasted work) but
+	// at the same time high enough to start listing before we process
+	// the whole page.
+	chanSize := 2 * workersNumber
+	if chanSize < 256 {
+		chanSize = 256
+	}
+	toProcess := make(chan runtime.Object, chanSize)
 	errs := make(chan error, workersNumber+1)
 	workersExited := make(chan struct{})
 
@@ -1183,8 +1188,8 @@ func (e *Store) DeleteCollection(ctx context.Context, deleteValidation rest.Vali
 			})
 			defer wg.Done()
 
-			for index := range toProcess {
-				accessor, err := meta.Accessor(items[index])
+			for item := range toProcess {
+				accessor, err := meta.Accessor(item)
 				if err != nil {
 					errs <- err
 					return
@@ -1210,20 +1215,82 @@ func (e *Store) DeleteCollection(ctx context.Context, deleteValidation rest.Vali
 		close(workersExited)
 	}()
 
-	func() {
+	hasLimit := listOptions.Limit > 0
+	if listOptions.Limit == 0 {
+		listOptions.Limit = deleteCollectionPageSize
+	}
+
+	// Paginate the list request and throw all items into workers.
+	listObj, err := func() (runtime.Object, error) {
 		defer close(toProcess)
 
-		for i := 0; i < len(items); i++ {
+		processedItems := 0
+		var originalList runtime.Object
+		for {
 			select {
-			case toProcess <- i:
-			case <-workersExited:
-				klog.V(4).InfoS("workers already exited, and there are some items waiting to be processed", "finished", i, "total", len(items))
-				return
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
 			}
+
+			listObj, err := e.List(ctx, listOptions)
+			if err != nil {
+				return nil, err
+			}
+
+			newItems, err := meta.ExtractList(listObj)
+			if err != nil {
+				return nil, err
+			}
+			items = append(items, newItems...)
+
+			for i := 0; i < len(newItems); i++ {
+				select {
+				case toProcess <- newItems[i]:
+				case <-workersExited:
+					klog.V(4).InfoS("workers already exited, and there are some items waiting to be processed", "queued/finished", i, "total", processedItems+len(newItems))
+					// Try to propagate an error from the workers if possible.
+					select {
+					case err := <-errs:
+						return nil, err
+					default:
+						return nil, fmt.Errorf("all DeleteCollection workers exited")
+					}
+				}
+			}
+			processedItems += len(newItems)
+
+			// If the original request was setting the limit, finish after running it.
+			if hasLimit {
+				return listObj, nil
+			}
+
+			if originalList == nil {
+				originalList = listObj
+				meta.SetList(originalList, nil)
+			}
+
+			// If there are no more items, return the list.
+			m, err := meta.ListAccessor(listObj)
+			if err != nil {
+				return nil, err
+			}
+			if len(m.GetContinue()) == 0 {
+				meta.SetList(originalList, items)
+				return originalList, nil
+			}
+
+			// Set up the next loop.
+			listOptions.Continue = m.GetContinue()
+			listOptions.ResourceVersion = ""
+			listOptions.ResourceVersionMatch = ""
 		}
 	}()
+	if err != nil {
+		return nil, err
+	}
 
-	// Wait for all workers to exist.
+	// Wait for all workers to exit.
 	<-workersExited
 
 	select {

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -65,6 +65,7 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
+	storagevalue "k8s.io/apiserver/pkg/storage/value"
 	"k8s.io/apiserver/pkg/storageversion"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
@@ -85,6 +86,13 @@ import (
 	_ "k8s.io/apiserver/pkg/apis/apiserver/install"
 )
 
+// hostnameFunc is a function to set the hostnameFunc of this apiserver.
+// To be used for testing purpose only, to simulate scenarios where multiple apiservers
+// exist. In such cases we want to ensure unique apiserver IDs which are a hash of hostnameFunc.
+var (
+	hostnameFunc = os.Hostname
+)
+
 const (
 	// DefaultLegacyAPIPrefix is where the legacy APIs will be located.
 	DefaultLegacyAPIPrefix = "/api"
@@ -190,6 +198,8 @@ type Config struct {
 	// SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true.
 	SkipOpenAPIInstallation bool
 
+	// ResourceTransformers are used to transform resources from and to etcd, e.g. encryption.
+	ResourceTransformers storagevalue.ResourceTransformers
 	// RESTOptionsGetter is used to construct RESTStorage types via the generic registry.
 	RESTOptionsGetter genericregistry.RESTOptionsGetter
 
@@ -364,7 +374,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 	defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
 	var id string
 	if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerIdentity) {
-		hostname, err := os.Hostname()
+		hostname, err := hostnameFunc()
 		if err != nil {
 			klog.Fatalf("error getting hostname for apiserver identity: %v", err)
 		}
@@ -894,14 +904,16 @@ func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c
 }
 
 func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
-	handler := filterlatency.TrackCompleted(apiHandler)
+	handler := apiHandler
+
+	handler = filterlatency.TrackCompleted(handler)
 	handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
 	handler = filterlatency.TrackStarted(handler, c.TracerProvider, "authorization")
 
 	if c.FlowControl != nil {
 		workEstimatorCfg := flowcontrolrequest.DefaultWorkEstimatorConfig()
 		requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
-			c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg)
+			c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats)
 		handler = filterlatency.TrackCompleted(handler)
 		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
 		handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
@@ -1067,3 +1079,12 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
 	tokenAuthenticator := authenticatorfactory.NewFromTokens(tokens, authn.APIAudiences)
 	authn.Authenticator = authenticatorunion.New(tokenAuthenticator, authn.Authenticator)
 }
+
+// For testing purpose only
+func SetHostnameFuncForTests(name string) {
+	hostnameFunc = func() (host string, err error) {
+		host = name
+		err = nil
+		return
+	}
+}

vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -34,7 +34,6 @@ import (
 
 const (
 	// Constant for the retry-after interval on rate limiting.
-	// TODO: maybe make this dynamic? or user-adjustable?
 	retryAfter = "1"
 
 	// How often inflight usage metric should be updated. Because
@@ -210,7 +209,7 @@ func WithMaxInFlightLimit(
 				// We need to split this data between buckets used for throttling.
 				metrics.RecordDroppedRequest(r, requestInfo, metrics.APIServerComponent, isMutatingRequest)
 				metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests)
-				tooManyRequests(r, w)
+				tooManyRequests(r, w, retryAfter)
 			}
 		}
 	})
@@ -221,9 +220,3 @@ func WithMaxInFlightLimit(
 func StartMaxInFlightWatermarkMaintenance(stopCh <-chan struct{}) {
 	startWatermarkMaintenance(watermark, stopCh)
 }
-
-func tooManyRequests(req *http.Request, w http.ResponseWriter) {
-	// Return a 429 status indicating "Too Many Requests"
-	w.Header().Set("Retry-After", retryAfter)
-	http.Error(w, "Too many requests, please try again later.", http.StatusTooManyRequests)
-}

vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net/http"
 	"runtime"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -67,6 +68,240 @@ func truncateLogField(s string) string {
 
 var initAPFOnce sync.Once
 
+type priorityAndFairnessHandler struct {
+	handler                 http.Handler
+	longRunningRequestCheck apirequest.LongRunningRequestCheck
+	fcIfc                   utilflowcontrol.Interface
+	workEstimator           flowcontrolrequest.WorkEstimatorFunc
+
+	// droppedRequests tracks the history of dropped requests for
+	// the purpose of computing RetryAfter header to avoid system
+	// overload.
+	droppedRequests utilflowcontrol.DroppedRequestsTracker
+}
+
+func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+	if !ok {
+		handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
+		return
+	}
+	user, ok := apirequest.UserFrom(ctx)
+	if !ok {
+		handleError(w, r, fmt.Errorf("no User found in context"))
+		return
+	}
+
+	isWatchRequest := watchVerbs.Has(requestInfo.Verb)
+
+	// Skip tracking long running non-watch requests.
+	if h.longRunningRequestCheck != nil && h.longRunningRequestCheck(r, requestInfo) && !isWatchRequest {
+		klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+
+	var classification *PriorityAndFairnessClassification
+	noteFn := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string) {
+		classification = &PriorityAndFairnessClassification{
+			FlowSchemaName:    fs.Name,
+			FlowSchemaUID:     fs.UID,
+			PriorityLevelName: pl.Name,
+			PriorityLevelUID:  pl.UID,
+		}
+
+		httplog.AddKeyValue(ctx, "apf_pl", truncateLogField(pl.Name))
+		httplog.AddKeyValue(ctx, "apf_fs", truncateLogField(fs.Name))
+	}
+	// estimateWork is called, if at all, after noteFn
+	estimateWork := func() flowcontrolrequest.WorkEstimate {
+		if classification == nil {
+			// workEstimator is being invoked before classification of
+			// the request has completed, we should never be here though.
+			klog.ErrorS(fmt.Errorf("workEstimator is being invoked before classification of the request has completed"),
+				"Using empty FlowSchema and PriorityLevelConfiguration name", "verb", r.Method, "URI", r.RequestURI)
+			return h.workEstimator(r, "", "")
+		}
+
+		workEstimate := h.workEstimator(r, classification.FlowSchemaName, classification.PriorityLevelName)
+
+		fcmetrics.ObserveWorkEstimatedSeats(classification.PriorityLevelName, classification.FlowSchemaName, workEstimate.MaxSeats())
+		httplog.AddKeyValue(ctx, "apf_iseats", workEstimate.InitialSeats)
+		httplog.AddKeyValue(ctx, "apf_fseats", workEstimate.FinalSeats)
+		httplog.AddKeyValue(ctx, "apf_additionalLatency", workEstimate.AdditionalLatency)
+
+		return workEstimate
+	}
+
+	var served bool
+	isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
+	noteExecutingDelta := func(delta int32) {
+		if isMutatingRequest {
+			watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
+		} else {
+			watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
+		}
+	}
+	noteWaitingDelta := func(delta int32) {
+		if isMutatingRequest {
+			waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
+		} else {
+			waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
+		}
+	}
+	queueNote := func(inQueue bool) {
+		if inQueue {
+			noteWaitingDelta(1)
+		} else {
+			noteWaitingDelta(-1)
+		}
+	}
+
+	digest := utilflowcontrol.RequestDigest{
+		RequestInfo: requestInfo,
+		User:        user,
+	}
+
+	if isWatchRequest {
+		// This channel blocks calling handler.ServeHTTP() until closed, and is closed inside execute().
+		// If APF rejects the request, it is never closed.
+		shouldStartWatchCh := make(chan struct{})
+
+		watchInitializationSignal := newInitializationSignal()
+		// This wraps the request passed to handler.ServeHTTP(),
+		// setting a context that plumbs watchInitializationSignal to storage
+		var watchReq *http.Request
+		// This is set inside execute(), prior to closing shouldStartWatchCh.
+		// If the request is rejected by APF it is left nil.
+		var forgetWatch utilflowcontrol.ForgetWatchFunc
+
+		defer func() {
+			// Protect from the situation when request will not reach storage layer
+			// and the initialization signal will not be send.
+			if watchInitializationSignal != nil {
+				watchInitializationSignal.Signal()
+			}
+			// Forget the watcher if it was registered.
+			//
+			// This is race-free because by this point, one of the following occurred:
+			// case <-shouldStartWatchCh: execute() completed the assignment to forgetWatch
+			// case <-resultCh: Handle() completed, and Handle() does not return
+			//   while execute() is running
+			if forgetWatch != nil {
+				forgetWatch()
+			}
+		}()
+
+		execute := func() {
+			startedAt := time.Now()
+			defer func() {
+				httplog.AddKeyValue(ctx, "apf_init_latency", time.Since(startedAt))
+			}()
+			noteExecutingDelta(1)
+			defer noteExecutingDelta(-1)
+			served = true
+			setResponseHeaders(classification, w)
+
+			forgetWatch = h.fcIfc.RegisterWatch(r)
+
+			// Notify the main thread that we're ready to start the watch.
+			close(shouldStartWatchCh)
+
+			// Wait until the request is finished from the APF point of view
+			// (which is when its initialization is done).
+			watchInitializationSignal.Wait()
+		}
+
+		// Ensure that an item can be put to resultCh asynchronously.
+		resultCh := make(chan interface{}, 1)
+
+		// Call Handle in a separate goroutine.
+		// The reason for it is that from APF point of view, the request processing
+		// finishes as soon as watch is initialized (which is generally orders of
+		// magnitude faster then the watch request itself). This means that Handle()
+		// call finishes much faster and for performance reasons we want to reduce
+		// the number of running goroutines - so we run the shorter thing in a
+		// dedicated goroutine and the actual watch handler in the main one.
+		go func() {
+			defer func() {
+				err := recover()
+				// do not wrap the sentinel ErrAbortHandler panic value
+				if err != nil && err != http.ErrAbortHandler {
+					// Same as stdlib http server code. Manually allocate stack
+					// trace buffer size to prevent excessively large logs
+					const size = 64 << 10
+					buf := make([]byte, size)
+					buf = buf[:runtime.Stack(buf, false)]
+					err = fmt.Sprintf("%v\n%s", err, buf)
+				}
+
+				// Ensure that the result is put into resultCh independently of the panic.
+				resultCh <- err
+			}()
+
+			// We create handleCtx with explicit cancelation function.
+			// The reason for it is that Handle() underneath may start additional goroutine
+			// that is blocked on context cancellation. However, from APF point of view,
+			// we don't want to wait until the whole watch request is processed (which is
+			// when it context is actually cancelled) - we want to unblock the goroutine as
+			// soon as the request is processed from the APF point of view.
+			//
+			// Note that we explicitly do NOT call the actuall handler using that context
+			// to avoid cancelling request too early.
+			handleCtx, handleCtxCancel := context.WithCancel(ctx)
+			defer handleCtxCancel()
+
+			// Note that Handle will return irrespective of whether the request
+			// executes or is rejected. In the latter case, the function will return
+			// without calling the passed `execute` function.
+			h.fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
+		}()
+
+		select {
+		case <-shouldStartWatchCh:
+			watchCtx := utilflowcontrol.WithInitializationSignal(ctx, watchInitializationSignal)
+			watchReq = r.WithContext(watchCtx)
+			h.handler.ServeHTTP(w, watchReq)
+			// Protect from the situation when request will not reach storage layer
+			// and the initialization signal will not be send.
+			// It has to happen before waiting on the resultCh below.
+			watchInitializationSignal.Signal()
+			// TODO: Consider finishing the request as soon as Handle call panics.
+			if err := <-resultCh; err != nil {
+				panic(err)
+			}
+		case err := <-resultCh:
+			if err != nil {
+				panic(err)
+			}
+		}
+	} else {
+		execute := func() {
+			noteExecutingDelta(1)
+			defer noteExecutingDelta(-1)
+			served = true
+			setResponseHeaders(classification, w)
+
+			h.handler.ServeHTTP(w, r)
+		}
+
+		h.fcIfc.Handle(ctx, digest, noteFn, estimateWork, queueNote, execute)
+	}
+
+	if !served {
+		setResponseHeaders(classification, w)
+
+		epmetrics.RecordDroppedRequest(r, requestInfo, epmetrics.APIServerComponent, isMutatingRequest)
+		epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
+		h.droppedRequests.RecordDroppedRequest(classification.PriorityLevelName)
+
+		// TODO(wojtek-t): Idea from deads2k: we can consider some jittering and in case of non-int
+		//  number, just return the truncated result and sleep the remainder server-side.
+		tooManyRequests(r, w, strconv.Itoa(int(h.droppedRequests.GetRetryAfter(classification.PriorityLevelName))))
+	}
+}
+
 // WithPriorityAndFairness limits the number of in-flight
 // requests in a fine-grained way.
 func WithPriorityAndFairness(
@@ -86,223 +321,15 @@ func WithPriorityAndFairness(
 		waitingMark.readOnlyObserver = fcmetrics.GetWaitingReadonlyConcurrency()
 		waitingMark.mutatingObserver = fcmetrics.GetWaitingMutatingConcurrency()
 	})
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
-		if !ok {
-			handleError(w, r, fmt.Errorf("no RequestInfo found in context"))
-			return
-		}
-		user, ok := apirequest.UserFrom(ctx)
-		if !ok {
-			handleError(w, r, fmt.Errorf("no User found in context"))
-			return
-		}
 
-		isWatchRequest := watchVerbs.Has(requestInfo.Verb)
-
-		// Skip tracking long running non-watch requests.
-		if longRunningRequestCheck != nil && longRunningRequestCheck(r, requestInfo) && !isWatchRequest {
-			klog.V(6).Infof("Serving RequestInfo=%#+v, user.Info=%#+v as longrunning\n", requestInfo, user)
-			handler.ServeHTTP(w, r)
-			return
-		}
-
-		var classification *PriorityAndFairnessClassification
-		noteFn := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, flowDistinguisher string) {
-			classification = &PriorityAndFairnessClassification{
-				FlowSchemaName:    fs.Name,
-				FlowSchemaUID:     fs.UID,
-				PriorityLevelName: pl.Name,
-				PriorityLevelUID:  pl.UID}
-
-			httplog.AddKeyValue(ctx, "apf_pl", truncateLogField(pl.Name))
-			httplog.AddKeyValue(ctx, "apf_fs", truncateLogField(fs.Name))
-		}
-		// estimateWork is called, if at all, after noteFn
-		estimateWork := func() flowcontrolrequest.WorkEstimate {
-			if classification == nil {
-				// workEstimator is being invoked before classification of
-				// the request has completed, we should never be here though.
-				klog.ErrorS(fmt.Errorf("workEstimator is being invoked before classification of the request has completed"),
-					"Using empty FlowSchema and PriorityLevelConfiguration name", "verb", r.Method, "URI", r.RequestURI)
-
-				return workEstimator(r, "", "")
-			}
-
-			workEstimate := workEstimator(r, classification.FlowSchemaName, classification.PriorityLevelName)
-
-			fcmetrics.ObserveWorkEstimatedSeats(classification.PriorityLevelName, classification.FlowSchemaName, workEstimate.MaxSeats())
-			httplog.AddKeyValue(ctx, "apf_iseats", workEstimate.InitialSeats)
-			httplog.AddKeyValue(ctx, "apf_fseats", workEstimate.FinalSeats)
-			httplog.AddKeyValue(ctx, "apf_additionalLatency", workEstimate.AdditionalLatency)
-
-			return workEstimate
-		}
-
-		var served bool
-		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
-		noteExecutingDelta := func(delta int32) {
-			if isMutatingRequest {
-				watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
-			} else {
-				watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
-			}
-		}
-		noteWaitingDelta := func(delta int32) {
-			if isMutatingRequest {
-				waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
-			} else {
-				waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
-			}
-		}
-		queueNote := func(inQueue bool) {
-			if inQueue {
-				noteWaitingDelta(1)
-			} else {
-				noteWaitingDelta(-1)
-			}
-		}
-
-		digest := utilflowcontrol.RequestDigest{
-			RequestInfo: requestInfo,
-			User:        user,
-		}
-
-		if isWatchRequest {
-			// This channel blocks calling handler.ServeHTTP() until closed, and is closed inside execute().
-			// If APF rejects the request, it is never closed.
-			shouldStartWatchCh := make(chan struct{})
-
-			watchInitializationSignal := newInitializationSignal()
-			// This wraps the request passed to handler.ServeHTTP(),
-			// setting a context that plumbs watchInitializationSignal to storage
-			var watchReq *http.Request
-			// This is set inside execute(), prior to closing shouldStartWatchCh.
-			// If the request is rejected by APF it is left nil.
-			var forgetWatch utilflowcontrol.ForgetWatchFunc
-
-			defer func() {
-				// Protect from the situation when request will not reach storage layer
-				// and the initialization signal will not be send.
-				if watchInitializationSignal != nil {
-					watchInitializationSignal.Signal()
-				}
-				// Forget the watcher if it was registered.
-				//
-				// // This is race-free because by this point, one of the following occurred:
-				// case <-shouldStartWatchCh: execute() completed the assignment to forgetWatch
-				// case <-resultCh: Handle() completed, and Handle() does not return
-				//   while execute() is running
-				if forgetWatch != nil {
-					forgetWatch()
-				}
-			}()
-
-			execute := func() {
-				startedAt := time.Now()
-				defer func() {
-					httplog.AddKeyValue(ctx, "apf_init_latency", time.Since(startedAt))
-				}()
-				noteExecutingDelta(1)
-				defer noteExecutingDelta(-1)
-				served = true
-				setResponseHeaders(classification, w)
-
-				forgetWatch = fcIfc.RegisterWatch(r)
-
-				// Notify the main thread that we're ready to start the watch.
-				close(shouldStartWatchCh)
-
-				// Wait until the request is finished from the APF point of view
-				// (which is when its initialization is done).
-				watchInitializationSignal.Wait()
-			}
-
-			// Ensure that an item can be put to resultCh asynchronously.
-			resultCh := make(chan interface{}, 1)
-
-			// Call Handle in a separate goroutine.
-			// The reason for it is that from APF point of view, the request processing
-			// finishes as soon as watch is initialized (which is generally orders of
-			// magnitude faster then the watch request itself). This means that Handle()
-			// call finishes much faster and for performance reasons we want to reduce
-			// the number of running goroutines - so we run the shorter thing in a
-			// dedicated goroutine and the actual watch handler in the main one.
-			go func() {
-				defer func() {
-					err := recover()
-					// do not wrap the sentinel ErrAbortHandler panic value
-					if err != nil && err != http.ErrAbortHandler {
-						// Same as stdlib http server code. Manually allocate stack
-						// trace buffer size to prevent excessively large logs
-						const size = 64 << 10
-						buf := make([]byte, size)
-						buf = buf[:runtime.Stack(buf, false)]
-						err = fmt.Sprintf("%v\n%s", err, buf)
-					}
-
-					// Ensure that the result is put into resultCh independently of the panic.
-					resultCh <- err
-				}()
-
-				// We create handleCtx with explicit cancelation function.
-				// The reason for it is that Handle() underneath may start additional goroutine
-				// that is blocked on context cancellation. However, from APF point of view,
-				// we don't want to wait until the whole watch request is processed (which is
-				// when it context is actually cancelled) - we want to unblock the goroutine as
-				// soon as the request is processed from the APF point of view.
-				//
-				// Note that we explicitly do NOT call the actuall handler using that context
-				// to avoid cancelling request too early.
-				handleCtx, handleCtxCancel := context.WithCancel(ctx)
-				defer handleCtxCancel()
-
-				// Note that Handle will return irrespective of whether the request
-				// executes or is rejected. In the latter case, the function will return
-				// without calling the passed `execute` function.
-				fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
-			}()
-
-			select {
-			case <-shouldStartWatchCh:
-				watchCtx := utilflowcontrol.WithInitializationSignal(ctx, watchInitializationSignal)
-				watchReq = r.WithContext(watchCtx)
-				handler.ServeHTTP(w, watchReq)
-				// Protect from the situation when request will not reach storage layer
-				// and the initialization signal will not be send.
-				// It has to happen before waiting on the resultCh below.
-				watchInitializationSignal.Signal()
-				// TODO: Consider finishing the request as soon as Handle call panics.
-				if err := <-resultCh; err != nil {
-					panic(err)
-				}
-			case err := <-resultCh:
-				if err != nil {
-					panic(err)
-				}
-			}
-		} else {
-			execute := func() {
-				noteExecutingDelta(1)
-				defer noteExecutingDelta(-1)
-				served = true
-				setResponseHeaders(classification, w)
-
-				handler.ServeHTTP(w, r)
-			}
-
-			fcIfc.Handle(ctx, digest, noteFn, estimateWork, queueNote, execute)
-		}
-
-		if !served {
-			setResponseHeaders(classification, w)
-
-			epmetrics.RecordDroppedRequest(r, requestInfo, epmetrics.APIServerComponent, isMutatingRequest)
-			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
-			tooManyRequests(r, w)
-		}
-	})
+	priorityAndFairnessHandler := &priorityAndFairnessHandler{
+		handler:                 handler,
+		longRunningRequestCheck: longRunningRequestCheck,
+		fcIfc:                   fcIfc,
+		workEstimator:           workEstimator,
+		droppedRequests:         utilflowcontrol.NewDroppedRequestsTracker(),
+	}
+	return http.HandlerFunc(priorityAndFairnessHandler.Handle)
 }
 
 // StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
@@ -323,3 +350,9 @@ func setResponseHeaders(classification *PriorityAndFairnessClassification, w htt
 	w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
 	w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
 }
+
+func tooManyRequests(req *http.Request, w http.ResponseWriter, retryAfter string) {
+	// Return a 429 status indicating "Too Many Requests"
+	w.Header().Set("Retry-After", retryAfter)
+	http.Error(w, "Too many requests, please try again later.", http.StatusTooManyRequests)
+}

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -18,6 +18,7 @@ package server
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	gpath "path"
@@ -736,16 +737,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}, shutdow
 }
 
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
-func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, openAPIModels map[string]*spec.Schema) error {
-	var typeConverter managedfields.TypeConverter
-
-	if len(openAPIModels) > 0 {
-		var err error
-		typeConverter, err = managedfields.NewTypeConverter(openAPIModels, false)
-		if err != nil {
-			return err
-		}
-	}
+func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, typeConverter managedfields.TypeConverter) error {
 	var resourceInfos []*storageversion.ResourceInfo
 	for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
 		if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
@@ -844,6 +836,9 @@ func (s *GenericAPIServer) InstallLegacyAPIGroup(apiPrefix string, apiGroupInfo
 // underlying storage will be destroyed on this servers shutdown.
 func (s *GenericAPIServer) InstallAPIGroups(apiGroupInfos ...*APIGroupInfo) error {
 	for _, apiGroupInfo := range apiGroupInfos {
+		if len(apiGroupInfo.PrioritizedVersions) == 0 {
+			return fmt.Errorf("no version priority set for %#v", *apiGroupInfo)
+		}
 		// Do not register empty group or empty version.  Doing so claims /apis/ for the wrong entity to be returned.
 		// Catching these here places the error  much closer to its origin
 		if len(apiGroupInfo.PrioritizedVersions[0].Group) == 0 {
@@ -916,9 +911,22 @@ func (s *GenericAPIServer) getAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV
 }
 
 func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupVersion schema.GroupVersion) *genericapi.APIGroupVersion {
+
+	allServedVersionsByResource := map[string][]string{}
+	for version, resourcesInVersion := range apiGroupInfo.VersionedResourcesStorageMap {
+		for resource := range resourcesInVersion {
+			if len(groupVersion.Group) == 0 {
+				allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], version)
+			} else {
+				allServedVersionsByResource[resource] = append(allServedVersionsByResource[resource], fmt.Sprintf("%s/%s", groupVersion.Group, version))
+			}
+		}
+	}
+
 	return &genericapi.APIGroupVersion{
-		GroupVersion:     groupVersion,
-		MetaGroupVersion: apiGroupInfo.MetaGroupVersion,
+		GroupVersion:                groupVersion,
+		AllServedVersionsByResource: allServedVersionsByResource,
+		MetaGroupVersion:            apiGroupInfo.MetaGroupVersion,
 
 		ParameterCodec:        apiGroupInfo.ParameterCodec,
 		Serializer:            apiGroupInfo.NegotiatedSerializer,
@@ -953,13 +961,13 @@ func NewDefaultAPIGroupInfo(group string, scheme *runtime.Scheme, parameterCodec
 }
 
 // getOpenAPIModels is a private method for getting the OpenAPI models
-func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (map[string]*spec.Schema, error) {
+func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*APIGroupInfo) (managedfields.TypeConverter, error) {
 	if s.openAPIV3Config == nil {
-		//!TODO: A future work should add a requirement that
-		// OpenAPIV3 config is required. May require some refactoring of tests.
-		return nil, nil
+		// SSA is GA and requires OpenAPI config to be set
+		// to create models.
+		return nil, errors.New("OpenAPIV3 config must not be nil")
 	}
-	pathsToIgnore := openapiutil.NewTrie(s.openAPIConfig.IgnorePrefixes)
+	pathsToIgnore := openapiutil.NewTrie(s.openAPIV3Config.IgnorePrefixes)
 	resourceNames := make([]string, 0)
 	for _, apiGroupInfo := range apiGroupInfos {
 		groupResources, err := getResourceNamesForGroup(apiPrefix, apiGroupInfo, pathsToIgnore)
@@ -977,7 +985,13 @@ func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*
 	for _, apiGroupInfo := range apiGroupInfos {
 		apiGroupInfo.StaticOpenAPISpec = openAPISpec
 	}
-	return openAPISpec, nil
+
+	typeConverter, err := managedfields.NewTypeConverter(openAPISpec, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return typeConverter, nil
 }
 
 // getResourceNamesForGroup is a private method for getting the canonical names for each resource to build in an api group

vendor/k8s.io/apiserver/pkg/server/handler.go

@@ -53,13 +53,13 @@ type APIServerHandler struct {
 	// Director is here so that we can properly handle fall through and proxy cases.
 	// This looks a bit bonkers, but here's what's happening.  We need to have /apis handling registered in gorestful in order to have
 	// swagger generated for compatibility.  Doing that with `/apis` as a webservice, means that it forcibly 404s (no defaulting allowed)
-	// all requests which are not /apis or /apis/.  We need those calls to fall through behind goresful for proper delegation.  Trying to
+	// all requests which are not /apis or /apis/.  We need those calls to fall through behind gorestful for proper delegation.  Trying to
 	// register for a pattern which includes everything behind it doesn't work because gorestful negotiates for verbs and content encoding
 	// and all those things go crazy when gorestful really just needs to pass through.  In addition, openapi enforces unique verb constraints
 	// which we don't fit into and it still muddies up swagger.  Trying to switch the webservices into a route doesn't work because the
 	//  containing webservice faces all the same problems listed above.
 	// This leads to the crazy thing done here.  Our mux does what we need, so we'll place it in front of gorestful.  It will introspect to
-	// decide if the route is likely to be handled by goresful and route there if needed.  Otherwise, it goes to NonGoRestfulMux mux in
+	// decide if the route is likely to be handled by gorestful and route there if needed.  Otherwise, it goes to NonGoRestfulMux mux in
 	// order to handle "normal" paths and delegation. Hopefully no API consumers will ever have to deal with this level of detail.  I think
 	// we should consider completely removing gorestful.
 	// Other servers should only use this opaquely to delegate to an API server.

vendor/k8s.io/apiserver/pkg/server/options/OWNERS

@@ -1,5 +1,7 @@
 # See the OWNERS docs at https://go.k8s.io/owners
 
+approvers:
+  - enj
 reviewers:
   - smarterclayton
   - wojtek-t

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -39,7 +39,6 @@ import (
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
 	"k8s.io/component-base/featuregate"
 )
 
@@ -123,7 +122,8 @@ func (a *AdmissionOptions) AddFlags(fs *pflag.FlagSet) {
 func (a *AdmissionOptions) ApplyTo(
 	c *server.Config,
 	informers informers.SharedInformerFactory,
-	kubeAPIServerClientConfig *rest.Config,
+	kubeClient kubernetes.Interface,
+	dynamicClient dynamic.Interface,
 	features featuregate.FeatureGate,
 	pluginInitializers ...admission.PluginInitializer,
 ) error {
@@ -143,15 +143,8 @@ func (a *AdmissionOptions) ApplyTo(
 		return fmt.Errorf("failed to read plugin config: %v", err)
 	}
 
-	clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
-	if err != nil {
-		return err
-	}
-	dynamicClient, err := dynamic.NewForConfig(kubeAPIServerClientConfig)
-	if err != nil {
-		return err
-	}
-	genericInitializer := initializer.New(clientset, dynamicClient, informers, c.Authorization.Authorizer, features, c.DrainedNotify())
+	genericInitializer := initializer.New(kubeClient, dynamicClient, informers, c.Authorization.Authorizer, features,
+		c.DrainedNotify())
 	initializersChain := admission.PluginInitializers{genericInitializer}
 	initializersChain = append(initializersChain, pluginInitializers...)
 

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -142,16 +142,6 @@ type AuditWebhookOptions struct {
 	GroupVersionString string
 }
 
-// AuditDynamicOptions control the configuration of dynamic backends for audit events
-type AuditDynamicOptions struct {
-	// Enabled tells whether the dynamic audit capability is enabled.
-	Enabled bool
-
-	// Configuration for batching backend. This is currently only used as an override
-	// for integration tests
-	BatchConfig *pluginbuffered.BatchConfig
-}
-
 func NewAuditOptions() *AuditOptions {
 	return &AuditOptions{
 		WebhookOptions: AuditWebhookOptions{

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -67,7 +67,7 @@ func (s *DeprecatedInsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	}
 
 	fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+
-		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening in all interfaces and IP families).")
+		"The IP address on which to serve the --insecure-port (set to 0.0.0.0 or :: for listening on all interfaces and IP address families).")
 	// Though this flag is deprecated, we discovered security concerns over how to do health checks without it e.g. #43784
 	fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.")
 	fs.Lookup("insecure-bind-address").Hidden = false
@@ -86,7 +86,7 @@ func (s *DeprecatedInsecureServingOptions) AddUnqualifiedFlags(fs *pflag.FlagSet
 	}
 
 	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
-		"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening in all interfaces and IP families).")
+		"The IP address on which to serve the insecure --port (set to '0.0.0.0' or '::' for listening on all interfaces and IP address families).")
 	fs.MarkDeprecated("address", "see --bind-address instead.")
 	fs.Lookup("address").Hidden = false
 

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -43,10 +43,11 @@ import (
 	"k8s.io/apiserver/pkg/apis/config/validation"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server/healthz"
-	"k8s.io/apiserver/pkg/storage/value"
+	storagevalue "k8s.io/apiserver/pkg/storage/value"
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
 	envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
+	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
@@ -63,13 +64,13 @@ const (
 	kmsTransformerPrefixV2       = "k8s:enc:kms:v2:"
 
 	// these constants relate to how the KMS v2 plugin status poll logic
-	// and the DEK generation logic behave.  In particular, the positive
+	// and the DEK/seed generation logic behave.  In particular, the positive
 	// interval and max TTL are closely related as the difference between
-	// these values defines the worst case window in which the write DEK
+	// these values defines the worst case window in which the write DEK/seed
 	// could expire due to the plugin going into an error state.  The
 	// worst case window divided by the negative interval defines the
 	// minimum amount of times the server will attempt to return to a
-	// healthy state before the DEK expires and writes begin to fail.
+	// healthy state before the DEK/seed expires and writes begin to fail.
 	//
 	// For now, these values are kept small and hardcoded to support being
 	// able to perform a "passive" storage migration while tolerating some
@@ -82,13 +83,13 @@ const (
 	// At that point, they are guaranteed to either migrate to the new key
 	// or get errors during the migration.
 	//
-	// If the API server coasted forever on the last DEK, they would need
+	// If the API server coasted forever on the last DEK/seed, they would need
 	// to actively check if it had observed the new key ID before starting
-	// a migration - otherwise it could keep using the old DEK and their
+	// a migration - otherwise it could keep using the old DEK/seed and their
 	// storage migration would not do what they thought it did.
 	kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
 	kmsv2PluginHealthzNegativeInterval = 10 * time.Second
-	kmsv2PluginWriteDEKMaxTTL          = 3 * time.Minute
+	kmsv2PluginWriteDEKSourceMaxTTL    = 3 * time.Minute
 
 	kmsPluginHealthzNegativeTTL = 3 * time.Second
 	kmsPluginHealthzPositiveTTL = 20 * time.Second
@@ -159,7 +160,7 @@ func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
 // EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
 type EncryptionConfiguration struct {
 	// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
-	Transformers map[schema.GroupResource]value.Transformer
+	Transformers map[schema.GroupResource]storagevalue.Transformer
 
 	// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
 	HealthChecks []healthz.HealthChecker
@@ -207,7 +208,7 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
 // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
 	var kmsHealthChecks []healthz.HealthChecker
 	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
 	if err != nil {
@@ -228,8 +229,8 @@ type healthChecker interface {
 // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
-	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
+func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
+	resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
 	var probes []healthChecker
 	var kmsUsed kmsState
 
@@ -268,11 +269,11 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
 		probes = append(probes, p...)
 	}
 
-	transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
+	transformers := make(map[schema.GroupResource]storagevalue.Transformer, len(resourceToPrefixTransformer))
 	for gr, transList := range resourceToPrefixTransformer {
 		gr := gr
 		transList := transList
-		transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
+		transformers[gr] = storagevalue.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
 	}
 
 	return transformers, probes, &kmsUsed, nil
@@ -332,8 +333,8 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
 	return nil
 }
 
-// rotateDEKOnKeyIDChange tries to rotate to a new DEK if the key ID returned by Status does not match the
-// current state.  If a successful rotation is performed, the new DEK and keyID overwrite the existing state.
+// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
+// current state.  If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
 // On any failure during rotation (including mismatch between status and encrypt calls), the current state is
 // preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
 // If the key ID returned by Status matches the current state, the expiration of the current state is extended
@@ -346,47 +347,62 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 
 	// allow reads indefinitely in all cases
 	// allow writes indefinitely as long as there is no error
-	// allow writes for only up to kmsv2PluginWriteDEKMaxTTL from now when there are errors
-	// we start the timer before we make the network call because kmsv2PluginWriteDEKMaxTTL is meant to be the upper bound
-	expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKMaxTTL)
+	// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
+	// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
+	expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)
 
-	// state is valid and status keyID is unchanged from when we generated this DEK so there is no need to rotate it
+	// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
+	// this gate can only change during tests, but the check is cheap enough to always make
+	// this allows us to easily exercise both modes without restarting the API server
+	// TODO integration test that this dynamically takes effect
+	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
+
+	// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
 	// just move the expiration of the current state forward by the reuse interval
-	if errState == nil && state.KeyID == statusKeyID {
+	// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
+	if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
 		state.ExpirationTimestamp = expirationTimestamp
 		h.state.Store(&state)
 		return nil
 	}
 
-	transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
+	transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)
 
-	if resp == nil {
-		resp = &kmsservice.EncryptResponse{} // avoid nil panics
+	if encObject == nil {
+		encObject = &kmstypes.EncryptedObject{} // avoid nil panics
 	}
 
 	// happy path, should be the common case
 	// TODO maybe add success metrics?
-	if errGen == nil && resp.KeyID == statusKeyID {
+	if errGen == nil && encObject.KeyID == statusKeyID {
 		h.state.Store(&envelopekmsv2.State{
 			Transformer:         transformer,
-			EncryptedDEK:        resp.Ciphertext,
-			KeyID:               resp.KeyID,
-			Annotations:         resp.Annotations,
+			EncryptedObject:     *encObject,
 			UID:                 uid,
 			ExpirationTimestamp: expirationTimestamp,
 			CacheKey:            cacheKey,
 		})
-		klog.V(6).InfoS("successfully rotated DEK",
-			"uid", uid,
-			"newKeyID", resp.KeyID,
-			"oldKeyID", state.KeyID,
-			"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
-		)
-		return nil
+
+		// it should be logically impossible for the new state to be invalid but check just in case
+		_, errGen = h.getCurrentState()
+		if errGen == nil {
+			klogV6 := klog.V(6)
+			if klogV6.Enabled() {
+				klogV6.InfoS("successfully rotated DEK",
+					"uid", uid,
+					"useSeed", useSeed,
+					"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
+					"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
+					"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
+				)
+			}
+			return nil
+		}
 	}
 
-	return fmt.Errorf("failed to rotate DEK uid=%q, errState=%v, errGen=%v, statusKeyID=%q, encryptKeyID=%q, stateKeyID=%q, expirationTimestamp=%s",
-		uid, errState, errGen, statusKeyID, resp.KeyID, state.KeyID, state.ExpirationTimestamp.Format(time.RFC3339))
+	return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
+		uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
 }
 
 // getCurrentState returns the latest state from the last status and encrypt calls.
@@ -399,12 +415,13 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
 		return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
 	}
 
-	if len(state.EncryptedDEK) == 0 {
-		return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty EncryptedDEK")
+	encryptedObjectCopy := state.EncryptedObject
+	if len(encryptedObjectCopy.EncryptedData) != 0 {
+		return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
 	}
-
-	if len(state.KeyID) == 0 {
-		return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty keyID")
+	encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
+	if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
+		return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
 	}
 
 	if state.ExpirationTimestamp.IsZero() {
@@ -429,7 +446,7 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 
 	if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
 		metrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
-		errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID %q: %w", response.KeyID, err))
+		errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
 	} else {
 		metrics.RecordKeyIDFromStatus(h.name, response.KeyID)
 		// unconditionally append as we filter out nil errors below
@@ -478,15 +495,15 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
 // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
-	var transformers []value.PrefixTransformer
+func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
+	var transformers []storagevalue.PrefixTransformer
 	var probes []healthChecker
 	var kmsUsed kmsState
 
 	for _, provider := range config.Providers {
 		provider := provider
 		var (
-			transformer    value.PrefixTransformer
+			transformer    storagevalue.PrefixTransformer
 			transformerErr error
 			probe          healthChecker
 			used           *kmsState
@@ -497,7 +514,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 			transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
 
 		case provider.AESCBC != nil:
-			cbcTransformer := func(block cipher.Block) (value.Transformer, error) {
+			cbcTransformer := func(block cipher.Block) (storagevalue.Transformer, error) {
 				return aestransformer.NewCBCTransformer(block), nil
 			}
 			transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, cbcTransformer, aesCBCTransformerPrefixV1)
@@ -513,7 +530,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 			}
 
 		case provider.Identity != nil:
-			transformer = value.PrefixTransformer{
+			transformer = storagevalue.PrefixTransformer{
 				Transformer: identity.NewEncryptCheckTransformer(),
 				Prefix:      []byte{},
 			}
@@ -532,10 +549,10 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 	return transformers, probes, &kmsUsed, nil
 }
 
-type blockTransformerFunc func(cipher.Block) (value.Transformer, error)
+type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error)
 
-func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
-	var result value.PrefixTransformer
+func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
+	var result storagevalue.PrefixTransformer
 
 	if len(config.Keys) == 0 {
 		return result, fmt.Errorf("aes provider has no valid keys")
@@ -550,7 +567,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
 		}
 	}
 
-	keyTransformers := []value.PrefixTransformer{}
+	keyTransformers := []storagevalue.PrefixTransformer{}
 
 	for _, keyData := range config.Keys {
 		keyData := keyData
@@ -569,26 +586,26 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
 
 		// Create a new PrefixTransformer for this key
 		keyTransformers = append(keyTransformers,
-			value.PrefixTransformer{
+			storagevalue.PrefixTransformer{
 				Transformer: transformer,
 				Prefix:      []byte(keyData.Name + ":"),
 			})
 	}
 
 	// Create a prefixTransformer which can choose between these keys
-	keyTransformer := value.NewPrefixTransformers(
+	keyTransformer := storagevalue.NewPrefixTransformers(
 		fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
 
 	// Create a PrefixTransformer which shall later be put in a list with other providers
-	result = value.PrefixTransformer{
+	result = storagevalue.PrefixTransformer{
 		Transformer: keyTransformer,
 		Prefix:      []byte(prefix),
 	}
 	return result, nil
 }
 
-func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
-	var result value.PrefixTransformer
+func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
+	var result storagevalue.PrefixTransformer
 
 	if len(config.Keys) == 0 {
 		return result, fmt.Errorf("secretbox provider has no valid keys")
@@ -603,7 +620,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 		}
 	}
 
-	keyTransformers := []value.PrefixTransformer{}
+	keyTransformers := []storagevalue.PrefixTransformer{}
 
 	for _, keyData := range config.Keys {
 		keyData := keyData
@@ -621,18 +638,18 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
 
 		// Create a new PrefixTransformer for this key
 		keyTransformers = append(keyTransformers,
-			value.PrefixTransformer{
+			storagevalue.PrefixTransformer{
 				Transformer: secretbox.NewSecretboxTransformer(keyArray),
 				Prefix:      []byte(keyData.Name + ":"),
 			})
 	}
 
 	// Create a prefixTransformer which can choose between these keys
-	keyTransformer := value.NewPrefixTransformers(
+	keyTransformer := storagevalue.NewPrefixTransformers(
 		fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
 
 	// Create a PrefixTransformer which shall later be put in a list with other providers
-	result = value.PrefixTransformer{
+	result = storagevalue.PrefixTransformer{
 		Transformer: keyTransformer,
 		Prefix:      []byte(secretboxTransformerPrefixV1),
 	}
@@ -665,13 +682,18 @@ func (s *kmsState) accumulate(other *kmsState) {
 // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (value.PrefixTransformer, healthChecker, *kmsState, error) {
+func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
 	kmsName := config.Name
 	switch config.APIVersion {
 	case kmsAPIVersionV1:
+		if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) {
+			return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.  Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.")
+		}
+		klog.InfoS("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.")
+
 		envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
 		if err != nil {
-			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
+			return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
 		}
 
 		probe := &kmsPluginProbe{
@@ -692,12 +714,12 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 
 	case kmsAPIVersionV2:
 		if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
-			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
+			return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
 		}
 
 		envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration)
 		if err != nil {
-			return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
+			return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
 		}
 
 		probe := &kmsv2PluginProbe{
@@ -710,45 +732,9 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 		// initialize state so that Load always works
 		probe.state.Store(&envelopekmsv2.State{})
 
-		runProbeCheckAndLog := func(ctx context.Context) error {
-			if err := probe.check(ctx); err != nil {
-				klog.VDepth(1, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
-				return err
-			}
-			return nil
-		}
+		primeAndProbeKMSv2(ctx, probe, kmsName)
 
-		// on the happy path where the plugin is healthy and available on server start,
-		// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
-		// ignore the error here since we want to support the plugin starting up async with the API server
-		_ = runProbeCheckAndLog(ctx)
-		// make sure that the plugin's key ID is reasonably up-to-date
-		// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
-		// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKMaxTTL
-		go wait.PollUntilWithContext(
-			ctx,
-			kmsv2PluginHealthzPositiveInterval,
-			func(ctx context.Context) (bool, error) {
-				if err := runProbeCheckAndLog(ctx); err == nil {
-					return false, nil
-				}
-
-				// TODO add integration test for quicker error poll on failure
-				// if we fail, block the outer polling and start a new quicker poll inline
-				// this limits the chance that our DEK expires during a transient failure
-				_ = wait.PollUntilWithContext(
-					ctx,
-					kmsv2PluginHealthzNegativeInterval,
-					func(ctx context.Context) (bool, error) {
-						return runProbeCheckAndLog(ctx) == nil, nil
-					},
-				)
-
-				return false, nil
-			})
-
-		// using AES-GCM by default for encrypting data with KMSv2
-		transformer := value.PrefixTransformer{
+		transformer := storagevalue.PrefixTransformer{
 			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState),
 			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
 		}
@@ -759,12 +745,62 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 		}, nil
 
 	default:
-		return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
+		return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
 	}
 }
 
-func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
-	baseTransformerFunc := func(block cipher.Block) (value.Transformer, error) {
+func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName string) {
+	runProbeCheckAndLog := func(ctx context.Context, depth int) error {
+		if err := probe.check(ctx); err != nil {
+			klog.VDepth(1+depth, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
+			return err
+		}
+		return nil
+	}
+
+	blockAndProbeFastUntilSuccess := func(ctx context.Context) {
+		_ = wait.PollUntilWithContext(
+			ctx,
+			kmsv2PluginHealthzNegativeInterval,
+			func(ctx context.Context) (bool, error) {
+				return runProbeCheckAndLog(ctx, 1) == nil, nil
+			},
+		)
+	}
+
+	// on the happy path where the plugin is healthy and available on server start,
+	// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
+	errPrime := runProbeCheckAndLog(ctx, 0)
+
+	// if our initial attempt to prime failed, start trying to get to a valid state in the background ASAP
+	// this prevents a slow start when the external healthz checker is configured to ignore the KMS healthz endpoint
+	// since we want to support the plugin starting up async with the API server, this error is not fatal
+	if errPrime != nil {
+		go blockAndProbeFastUntilSuccess(ctx) // separate go routine to avoid blocking
+	}
+
+	// make sure that the plugin's key ID is reasonably up-to-date
+	// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
+	// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
+	go wait.PollUntilWithContext(
+		ctx,
+		kmsv2PluginHealthzPositiveInterval,
+		func(ctx context.Context) (bool, error) {
+			if err := runProbeCheckAndLog(ctx, 0); err == nil {
+				return false, nil
+			}
+
+			// TODO add integration test for quicker error poll on failure
+			// if we fail, block the outer polling and start a new quicker poll inline
+			// this limits the chance that our DEK expires during a transient failure
+			blockAndProbeFastUntilSuccess(ctx)
+
+			return false, nil
+		})
+}
+
+func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
+	baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) {
 		gcm, err := aestransformer.NewGCMTransformer(block)
 		if err != nil {
 			return nil, err
@@ -777,15 +813,15 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop
 		return unionTransformers{gcm, aestransformer.NewCBCTransformer(block)}, nil
 	}
 
-	return value.PrefixTransformer{
+	return storagevalue.PrefixTransformer{
 		Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
 		Prefix:      []byte(prefix + config.Name + ":"),
 	}
 }
 
-type unionTransformers []value.Transformer
+type unionTransformers []storagevalue.Transformer
 
-func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
+func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, stale bool, err error) {
 	var errs []error
 	for i := range u {
 		transformer := u[i]
@@ -804,7 +840,7 @@ func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte
 	return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
 }
 
-func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
+func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, err error) {
 	return u[0].TransformToStorage(ctx, data, dataCtx)
 }
 
@@ -815,7 +851,7 @@ func computeEncryptionConfigHash(data []byte) string {
 	return fmt.Sprintf("%x", sha256.Sum256(data))
 }
 
-var _ ResourceTransformers = &DynamicTransformers{}
+var _ storagevalue.ResourceTransformers = &DynamicTransformers{}
 var _ healthz.HealthChecker = &DynamicTransformers{}
 
 // DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
@@ -825,7 +861,7 @@ type DynamicTransformers struct {
 }
 
 type transformTracker struct {
-	transformerOverrides  map[schema.GroupResource]value.Transformer
+	transformerOverrides  map[schema.GroupResource]storagevalue.Transformer
 	kmsPluginHealthzCheck healthz.HealthChecker
 	closeTransformers     context.CancelFunc
 	kmsCloseGracePeriod   time.Duration
@@ -833,7 +869,7 @@ type transformTracker struct {
 
 // NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
 func NewDynamicTransformers(
-	transformerOverrides map[schema.GroupResource]value.Transformer,
+	transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
 	kmsPluginHealthzCheck healthz.HealthChecker,
 	closeTransformers context.CancelFunc,
 	kmsCloseGracePeriod time.Duration,
@@ -864,7 +900,7 @@ func (d *DynamicTransformers) Name() string {
 }
 
 // TransformerForResource returns the transformer for the given resource.
-func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
+func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
 	return &resourceTransformer{
 		resource:         resource,
 		transformTracker: d.transformTracker,
@@ -873,7 +909,7 @@ func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResour
 
 // Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
 func (d *DynamicTransformers) Set(
-	transformerOverrides map[schema.GroupResource]value.Transformer,
+	transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
 	closeTransformers context.CancelFunc,
 	kmsPluginHealthzCheck healthz.HealthChecker,
 	kmsCloseGracePeriod time.Duration,
@@ -898,34 +934,30 @@ func (d *DynamicTransformers) Set(
 	}()
 }
 
-var _ value.Transformer = &resourceTransformer{}
+var _ storagevalue.Transformer = &resourceTransformer{}
 
 type resourceTransformer struct {
 	resource         schema.GroupResource
 	transformTracker *atomic.Value
 }
 
-func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
+func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) {
 	return r.transformer().TransformFromStorage(ctx, data, dataCtx)
 }
 
-func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
+func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) {
 	return r.transformer().TransformToStorage(ctx, data, dataCtx)
 }
 
-func (r *resourceTransformer) transformer() value.Transformer {
+func (r *resourceTransformer) transformer() storagevalue.Transformer {
 	return transformerFromOverrides(r.transformTracker.Load().(*transformTracker).transformerOverrides, r.resource)
 }
 
-type ResourceTransformers interface {
-	TransformerForResource(resource schema.GroupResource) value.Transformer
-}
+var _ storagevalue.ResourceTransformers = &StaticTransformers{}
 
-var _ ResourceTransformers = &StaticTransformers{}
+type StaticTransformers map[schema.GroupResource]storagevalue.Transformer
 
-type StaticTransformers map[schema.GroupResource]value.Transformer
-
-func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
+func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
 	return transformerFromOverrides(s, resource)
 }
 
@@ -934,7 +966,7 @@ var anyGroupAnyResource = schema.GroupResource{
 	Resource: "*",
 }
 
-func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer {
+func transformerFromOverrides(transformerOverrides map[schema.GroupResource]storagevalue.Transformer, resource schema.GroupResource) storagevalue.Transformer {
 	if transformer := transformerOverrides[resource]; transformer != nil {
 		return transformer
 	}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 )
@@ -163,16 +164,19 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	ctx, closeTransformers := context.WithCancel(serverCtx)
 
 	defer func() {
-		// TODO: increment success metric when updatedEffectiveConfig=true
-
 		// TODO can work queue metrics help here?
 
 		if !updatedEffectiveConfig {
 			// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
 			closeTransformers()
 		}
+
+		if updatedEffectiveConfig && err == nil {
+			metrics.RecordEncryptionConfigAutomaticReloadSuccess()
+		}
+
 		if err != nil {
-			// TODO: increment failure metric
+			metrics.RecordEncryptionConfigAutomaticReloadFailure()
 			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
 			// add dummy item back to the queue to trigger file content processing.
 			d.queue.AddRateLimited(key)

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "encryption_config_controller"
+)
+
+var (
+	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_failures_total",
+			Help:           "Total number of failed automatic reloads of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_success_total",
+			Help:           "Total number of successful automatic reloads of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
+
+	encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "automatic_reload_last_timestamp_seconds",
+			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+)
+
+var registerMetrics sync.Once
+
+func RegisterMetrics() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
+		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
+	})
+}
+
+func RecordEncryptionConfigAutomaticReloadFailure() {
+	encryptionConfigAutomaticReloadFailureTotal.Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("failure")
+}
+
+func RecordEncryptionConfigAutomaticReloadSuccess() {
+	encryptionConfigAutomaticReloadSuccessTotal.Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("success")
+}
+
+func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
+	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
+}

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -36,9 +36,10 @@ import (
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
 	encryptionconfigcontroller "k8s.io/apiserver/pkg/server/options/encryptionconfig/controller"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
-	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
+	storagevalue "k8s.io/apiserver/pkg/storage/value"
 	"k8s.io/klog/v2"
 )
 
@@ -64,11 +65,6 @@ type EtcdOptions struct {
 	// WatchCacheSizes represents override to a given resource
 	WatchCacheSizes []string
 
-	// complete guards fields that must be initialized via Complete before the Apply methods can be used.
-	complete               bool
-	resourceTransformers   encryptionconfig.ResourceTransformers
-	kmsPluginHealthzChecks []healthz.HealthChecker
-
 	// SkipHealthEndpoints, when true, causes the Apply methods to not set up health endpoints.
 	// This allows multiple invocations of the Apply methods without duplication of said endpoints.
 	SkipHealthEndpoints bool
@@ -212,92 +208,18 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 		"The time in seconds that each lease is reused. A lower value could avoid large number of objects reusing the same lease. Notice that a too small value may cause performance problems at storage layer.")
 }
 
-// Complete must be called exactly once before using any of the Apply methods.  It is responsible for setting
-// up objects that must be created once and reused across multiple invocations such as storage transformers.
-// This method mutates the receiver (EtcdOptions).  It must never mutate the inputs.
-func (s *EtcdOptions) Complete(
-	storageObjectCountTracker flowcontrolrequest.StorageObjectCountTracker,
-	stopCh <-chan struct{},
-	addPostStartHook func(name string, hook server.PostStartHookFunc) error,
-) error {
-	if s == nil {
-		return nil
-	}
-
-	if s.complete {
-		return fmt.Errorf("EtcdOptions.Complete called more than once")
-	}
-
-	if len(s.EncryptionProviderConfigFilepath) != 0 {
-		ctxServer := wait.ContextForChannel(stopCh)
-		// nolint:govet // The only code path where closeTransformers does not get called is when it gets stored in dynamicTransformers.
-		ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
-
-		encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload)
-		if err != nil {
-			// in case of error, we want to close partially initialized (if any) transformers
-			closeTransformers()
-			return err
-		}
-
-		// enable kms hot reload controller only if the config file is set to be automatically reloaded
-		if s.EncryptionProviderConfigAutomaticReload {
-			// with reload=true we will always have 1 health check
-			if len(encryptionConfiguration.HealthChecks) != 1 {
-				// in case of error, we want to close partially initialized (if any) transformers
-				closeTransformers()
-				return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
-			}
-
-			// Here the dynamic transformers take ownership of the transformers and their cancellation.
-			dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
-
-			// add post start hook to start hot reload controller
-			// adding this hook here will ensure that it gets configured exactly once
-			err = addPostStartHook(
-				"start-encryption-provider-config-automatic-reload",
-				func(_ server.PostStartHookContext) error {
-					dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration(
-						"encryption-provider-config-automatic-reload-controller",
-						s.EncryptionProviderConfigFilepath,
-						dynamicTransformers,
-						encryptionConfiguration.EncryptionFileContentHash,
-					)
-
-					go dynamicEncryptionConfigController.Run(ctxServer)
-
-					return nil
-				},
-			)
-			if err != nil {
-				// in case of error, we want to close partially initialized (if any) transformers
-				closeTransformers()
-				return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
-			}
-
-			s.resourceTransformers = dynamicTransformers
-			s.kmsPluginHealthzChecks = []healthz.HealthChecker{dynamicTransformers}
-		} else {
-			s.resourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
-			s.kmsPluginHealthzChecks = encryptionConfiguration.HealthChecks
-		}
-	}
-
-	s.StorageConfig.StorageObjectCountTracker = storageObjectCountTracker
-
-	s.complete = true
-
-	// nolint:govet // The only code path where closeTransformers does not get called is when it gets stored in dynamicTransformers.
-	return nil
-}
-
 // ApplyTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
 func (s *EtcdOptions) ApplyTo(c *server.Config) error {
 	if s == nil {
 		return nil
 	}
 
-	return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: s.StorageConfig}, c)
+	storageConfigCopy := s.StorageConfig
+	if storageConfigCopy.StorageObjectCountTracker == nil {
+		storageConfigCopy.StorageObjectCountTracker = c.StorageObjectCountTracker
+	}
+
+	return s.ApplyWithStorageFactoryTo(&SimpleStorageFactory{StorageConfig: storageConfigCopy}, c)
 }
 
 // ApplyWithStorageFactoryTo mutates the provided server.Config.  It must never mutate the receiver (EtcdOptions).
@@ -306,24 +228,118 @@ func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFac
 		return nil
 	}
 
-	if !s.complete {
-		return fmt.Errorf("EtcdOptions.Apply called without completion")
-	}
-
 	if !s.SkipHealthEndpoints {
 		if err := s.addEtcdHealthEndpoint(c); err != nil {
 			return err
 		}
 	}
 
-	if s.resourceTransformers != nil {
+	// setup encryption
+	if err := s.maybeApplyResourceTransformers(c); err != nil {
+		return err
+	}
+
+	metrics.SetStorageMonitorGetter(monitorGetter(factory))
+
+	c.RESTOptionsGetter = s.CreateRESTOptionsGetter(factory, c.ResourceTransformers)
+	return nil
+}
+
+func monitorGetter(factory serverstorage.StorageFactory) func() (monitors []metrics.Monitor, err error) {
+	return func() (monitors []metrics.Monitor, err error) {
+		defer func() {
+			if err != nil {
+				for _, m := range monitors {
+					m.Close()
+				}
+			}
+		}()
+
+		var m metrics.Monitor
+		for _, cfg := range factory.Configs() {
+			m, err = storagefactory.CreateMonitor(cfg)
+			if err != nil {
+				return nil, err
+			}
+			monitors = append(monitors, m)
+		}
+		return monitors, nil
+	}
+}
+
+func (s *EtcdOptions) CreateRESTOptionsGetter(factory serverstorage.StorageFactory, resourceTransformers storagevalue.ResourceTransformers) generic.RESTOptionsGetter {
+	if resourceTransformers != nil {
 		factory = &transformerStorageFactory{
 			delegate:             factory,
-			resourceTransformers: s.resourceTransformers,
+			resourceTransformers: resourceTransformers,
+		}
+	}
+	return &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
+}
+
+func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err error) {
+	if c.ResourceTransformers != nil {
+		return nil
+	}
+	if len(s.EncryptionProviderConfigFilepath) == 0 {
+		return nil
+	}
+
+	ctxServer := wait.ContextForChannel(c.DrainedNotify())
+	ctxTransformers, closeTransformers := context.WithCancel(ctxServer)
+	defer func() {
+		// in case of error, we want to close partially initialized (if any) transformers
+		if err != nil {
+			closeTransformers()
+		}
+	}()
+
+	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload)
+	if err != nil {
+		return err
+	}
+
+	if s.EncryptionProviderConfigAutomaticReload {
+		// with reload=true we will always have 1 health check
+		if len(encryptionConfiguration.HealthChecks) != 1 {
+			return fmt.Errorf("failed to start kms encryption config hot reload controller. only 1 health check should be available when reload is enabled")
+		}
+
+		// Here the dynamic transformers take ownership of the transformers and their cancellation.
+		dynamicTransformers := encryptionconfig.NewDynamicTransformers(encryptionConfiguration.Transformers, encryptionConfiguration.HealthChecks[0], closeTransformers, encryptionConfiguration.KMSCloseGracePeriod)
+
+		// add post start hook to start hot reload controller
+		// adding this hook here will ensure that it gets configured exactly once
+		err = c.AddPostStartHook(
+			"start-encryption-provider-config-automatic-reload",
+			func(_ server.PostStartHookContext) error {
+				dynamicEncryptionConfigController := encryptionconfigcontroller.NewDynamicEncryptionConfiguration(
+					"encryption-provider-config-automatic-reload-controller",
+					s.EncryptionProviderConfigFilepath,
+					dynamicTransformers,
+					encryptionConfiguration.EncryptionFileContentHash,
+				)
+
+				go dynamicEncryptionConfigController.Run(ctxServer)
+
+				return nil
+			},
+		)
+		if err != nil {
+			return fmt.Errorf("failed to add post start hook for kms encryption config hot reload controller: %w", err)
+		}
+
+		c.ResourceTransformers = dynamicTransformers
+		if !s.SkipHealthEndpoints {
+			c.AddHealthChecks(dynamicTransformers)
+		}
+	} else {
+		c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
+		if !s.SkipHealthEndpoints {
+			c.AddHealthChecks(encryptionConfiguration.HealthChecks...)
 		}
 	}
 
-	c.RESTOptionsGetter = &StorageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory}
 	return nil
 }
 
@@ -344,8 +360,6 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 		return readyCheck()
 	}))
 
-	c.AddHealthChecks(s.kmsPluginHealthzChecks...)
-
 	return nil
 }
 
@@ -457,7 +471,7 @@ var _ serverstorage.StorageFactory = &transformerStorageFactory{}
 
 type transformerStorageFactory struct {
 	delegate             serverstorage.StorageFactory
-	resourceTransformers encryptionconfig.ResourceTransformers
+	resourceTransformers storagevalue.ResourceTransformers
 }
 
 func (t *transformerStorageFactory) NewConfig(resource schema.GroupResource) (*storagebackend.ConfigForResource, error) {

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 
 	"github.com/spf13/pflag"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/features"
@@ -28,6 +27,7 @@ import (
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/featuregate"
 	"k8s.io/klog/v2"
@@ -101,9 +101,6 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
 // ApplyTo adds RecommendedOptions to the server configuration.
 // pluginInitializers can be empty, it is only need for additional initializers.
 func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
-	if err := o.Etcd.Complete(config.Config.StorageObjectCountTracker, config.Config.DrainedNotify(), config.Config.AddPostStartHook); err != nil {
-		return err
-	}
 	if err := o.Etcd.ApplyTo(&config.Config); err != nil {
 		return err
 	}
@@ -131,9 +128,20 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
-	if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
+	initializers, err := o.ExtraAdmissionInitializers(config)
+	if err != nil {
 		return err
-	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
+	}
+	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
+	if err != nil {
+		return err
+	}
+	dynamicClient, err := dynamic.NewForConfig(config.ClientConfig)
+	if err != nil {
+		return err
+	}
+	if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, kubeClient, dynamicClient, o.FeatureGate,
+		initializers...); err != nil {
 		return err
 	}
 	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -153,7 +153,7 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IPVar(&s.BindAddress, "bind-address", s.BindAddress, ""+
 		"The IP address on which to listen for the --secure-port port. The "+
 		"associated interface(s) must be reachable by the rest of the cluster, and by CLI/web "+
-		"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces will be used.")
+		"clients. If blank or an unspecified address (0.0.0.0 or ::), all interfaces and IP address families will be used.")
 
 	desc := "The port on which to serve HTTPS with authentication and authorization."
 	if s.Required {

vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -22,6 +22,7 @@ import (
 	cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics"
 	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	flowcontrolmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	peerproxymetrics "k8s.io/apiserver/pkg/util/peerproxy/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
@@ -50,4 +51,5 @@ func register() {
 	cachermetrics.Register()
 	etcd3metrics.Register()
 	flowcontrolmetrics.Register()
+	peerproxymetrics.Register()
 }

vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -43,10 +43,7 @@ func (oa OpenAPI) InstallV2(c *restful.Container, mux *mux.PathRecorderMux) (*ha
 	}
 	spec.Definitions = handler.PruneDefaults(spec.Definitions)
 	openAPIVersionedService := handler.NewOpenAPIService(spec)
-	err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
-	if err != nil {
-		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
-	}
+	openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
 
 	return openAPIVersionedService, spec
 }

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -291,28 +291,17 @@ func Configs(storageConfig storagebackend.Config) []storagebackend.Config {
 
 // Returns all storage configurations including those for group resource overrides
 func configs(storageConfig storagebackend.Config, grOverrides map[schema.GroupResource]groupResourceOverrides) []storagebackend.Config {
-	locations := sets.NewString()
-	configs := []storagebackend.Config{}
-	for _, loc := range storageConfig.Transport.ServerList {
-		// copy
-		newConfig := storageConfig
-		newConfig.Transport.ServerList = []string{loc}
-		configs = append(configs, newConfig)
-		locations.Insert(loc)
-	}
+	configs := []storagebackend.Config{storageConfig}
 
 	for _, override := range grOverrides {
-		for _, loc := range override.etcdLocation {
-			if locations.Has(loc) {
-				continue
-			}
-			// copy
-			newConfig := storageConfig
-			override.Apply(&newConfig, &StorageCodecConfig{})
-			newConfig.Transport.ServerList = []string{loc}
-			configs = append(configs, newConfig)
-			locations.Insert(loc)
+		if len(override.etcdLocation) == 0 {
+			continue
 		}
+		// copy
+		newConfig := storageConfig
+		override.Apply(&newConfig, &StorageCodecConfig{})
+		newConfig.Transport.ServerList = override.etcdLocation
+		configs = append(configs, newConfig)
 	}
 	return configs
 }

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -1,11 +1,9 @@
 # See the OWNERS docs at https://go.k8s.io/owners
 
 approvers:
-  - lavalamp
   - liggitt
   - wojtek-t
 reviewers:
-  - lavalamp
   - smarterclayton
   - wojtek-t
   - deads2k
@@ -16,6 +14,8 @@ reviewers:
   - ingvagabund
   - enj
   - stevekuznetsov
+  - MadhavJivrajani
 emeritus_approvers:
   - xiang90
   - timothysc
+  - lavalamp

vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -104,7 +104,7 @@ type Config struct {
 
 	Codec runtime.Codec
 
-	Clock clock.Clock
+	Clock clock.WithTicker
 }
 
 type watchersMap map[int]*cacheWatcher
@@ -184,7 +184,6 @@ func (i *indexedWatchers) terminateAll(groupResource schema.GroupResource, done
 // second in a bucket, and pop up them once at the timeout. To be more specific,
 // if you set fire time at X, you can get the bookmark within (X-1,X+1) period.
 type watcherBookmarkTimeBuckets struct {
-	lock sync.Mutex
 	// the key of watcherBuckets is the number of seconds since createTime
 	watchersBuckets   map[int64][]*cacheWatcher
 	createTime        time.Time
@@ -205,7 +204,7 @@ func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *
 
 // adds a watcher to the bucket, if the deadline is before the start, it will be
 // added to the first one.
-func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
+func (t *watcherBookmarkTimeBuckets) addWatcherThreadUnsafe(w *cacheWatcher) bool {
 	// note that the returned time can be before t.createTime,
 	// especially in cases when the nextBookmarkTime method
 	// give us the zero value of type Time
@@ -215,8 +214,6 @@ func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
 		return false
 	}
 	bucketID := int64(nextTime.Sub(t.createTime) / time.Second)
-	t.lock.Lock()
-	defer t.lock.Unlock()
 	if bucketID < t.startBucketID {
 		bucketID = t.startBucketID
 	}
@@ -225,12 +222,10 @@ func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
 	return true
 }
 
-func (t *watcherBookmarkTimeBuckets) popExpiredWatchers() [][]*cacheWatcher {
+func (t *watcherBookmarkTimeBuckets) popExpiredWatchersThreadUnsafe() [][]*cacheWatcher {
 	currentBucketID := int64(t.clock.Since(t.createTime) / time.Second)
 	// There should be one or two elements in almost all cases
 	expiredWatchers := make([][]*cacheWatcher, 0, 2)
-	t.lock.Lock()
-	defer t.lock.Unlock()
 	for ; t.startBucketID <= currentBucketID; t.startBucketID++ {
 		if watchers, ok := t.watchersBuckets[t.startBucketID]; ok {
 			delete(t.watchersBuckets, t.startBucketID)
@@ -328,11 +323,16 @@ type Cacher struct {
 	// dispatching that event to avoid race with closing channels in watchers.
 	watchersToStop []*cacheWatcher
 	// Maintain a timeout queue to send the bookmark event before the watcher times out.
+	// Note that this field when accessed MUST be protected by the Cacher.lock.
 	bookmarkWatchers *watcherBookmarkTimeBuckets
 	// expiredBookmarkWatchers is a list of watchers that were expired and need to be schedule for a next bookmark event
 	expiredBookmarkWatchers []*cacheWatcher
 }
 
+func (c *Cacher) RequestWatchProgress(ctx context.Context) error {
+	return c.storage.RequestWatchProgress(ctx)
+}
+
 // NewCacherFromConfig creates a new Cacher responsible for servicing WATCH and LIST requests from
 // its internal cache and updating its cache in the background based on the
 // given configuration.
@@ -401,10 +401,10 @@ func NewCacherFromConfig(config Config) (*Cacher, error) {
 		// so that future reuse does not get a spurious timeout.
 		<-cacher.timer.C
 	}
-
+	progressRequester := newConditionalProgressRequester(config.Storage.RequestWatchProgress, config.Clock)
 	watchCache := newWatchCache(
-		config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers, config.Clock, config.GroupResource)
-	listerWatcher := NewCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
+		config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers, config.Clock, config.GroupResource, progressRequester)
+	listerWatcher := NewListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
 	reflectorName := "storage/cacher.go:" + config.ResourcePrefix
 
 	reflector := cache.NewNamedReflector(reflectorName, listerWatcher, obj, watchCache, 0)
@@ -423,6 +423,7 @@ func NewCacherFromConfig(config Config) (*Cacher, error) {
 	cacher.reflector = reflector
 
 	go cacher.dispatchEvents()
+	go progressRequester.Run(stopCh)
 
 	cacher.stopWg.Add(1)
 	go func() {
@@ -592,6 +593,18 @@ func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions
 		identifier,
 	)
 
+	// note that c.waitUntilWatchCacheFreshAndForceAllEvents must be called without
+	// the c.watchCache.RLock held otherwise we are at risk of a deadlock
+	// mainly because c.watchCache.processEvent method won't be able to make progress
+	//
+	// moreover even though the c.waitUntilWatchCacheFreshAndForceAllEvents acquires a lock
+	// it is safe to release the lock after the method finishes because we don't require
+	// any atomicity between the call to the method and further calls that actually get the events.
+	forceAllEvents, err := c.waitUntilWatchCacheFreshAndForceAllEvents(ctx, requestedWatchRV, opts)
+	if err != nil {
+		return newErrWatcher(err), nil
+	}
+
 	// We explicitly use thread unsafe version and do locking ourself to ensure that
 	// no new events will be processed in the meantime. The watchCache will be unlocked
 	// on return from this function.
@@ -599,10 +612,7 @@ func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions
 	// underlying watchCache is calling processEvent under its lock.
 	c.watchCache.RLock()
 	defer c.watchCache.RUnlock()
-	forceAllEvents, err := c.waitUntilWatchCacheFreshAndForceAllEvents(ctx, requestedWatchRV, opts)
-	if err != nil {
-		return newErrWatcher(err), nil
-	}
+
 	startWatchRV := startWatchResourceVersionFn()
 	var cacheInterval *watchCacheInterval
 	if forceAllEvents {
@@ -638,7 +648,7 @@ func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions
 
 		// Add it to the queue only when the client support watch bookmarks.
 		if watcher.allowWatchBookmarks {
-			c.bookmarkWatchers.addWatcher(watcher)
+			c.bookmarkWatchers.addWatcherThreadUnsafe(watcher)
 		}
 		c.watcherIdx++
 	}()
@@ -716,17 +726,18 @@ func shouldDelegateList(opts storage.ListOptions) bool {
 	pred := opts.Predicate
 	match := opts.ResourceVersionMatch
 	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
+	consistentListFromCacheEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache)
+
+	// Serve consistent reads from storage if ConsistentListFromCache is disabled
+	consistentReadFromStorage := resourceVersion == "" && !consistentListFromCacheEnabled
+	// Watch cache doesn't support continuations, so serve them from etcd.
 	hasContinuation := pagingEnabled && len(pred.Continue) > 0
+	// Serve paginated requests about revision "0" from watch cache to avoid overwhelming etcd.
 	hasLimit := pagingEnabled && pred.Limit > 0 && resourceVersion != "0"
+	// Watch cache only supports ResourceVersionMatchNotOlderThan (default).
 	unsupportedMatch := match != "" && match != metav1.ResourceVersionMatchNotOlderThan
 
-	// If resourceVersion is not specified, serve it from underlying
-	// storage (for backward compatibility). If a continuation is
-	// requested, serve it from the underlying storage as well.
-	// Limits are only sent to storage when resourceVersion is non-zero
-	// since the watch cache isn't able to perform continuations, and
-	// limits are ignored when resource version is zero
-	return resourceVersion == "" || hasContinuation || hasLimit || unsupportedMatch
+	return consistentReadFromStorage || hasContinuation || hasLimit || unsupportedMatch
 }
 
 func (c *Cacher) listItems(ctx context.Context, listRV uint64, key string, pred storage.SelectionPredicate, recursive bool) ([]interface{}, uint64, string, error) {
@@ -752,19 +763,21 @@ func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptio
 		return c.storage.GetList(ctx, key, opts, listObj)
 	}
 
-	// If resourceVersion is specified, serve it from cache.
-	// It's guaranteed that the returned value is at least that
-	// fresh as the given resourceVersion.
 	listRV, err := c.versioner.ParseResourceVersion(resourceVersion)
 	if err != nil {
 		return err
 	}
-
 	if listRV == 0 && !c.ready.check() {
 		// If Cacher is not yet initialized and we don't require any specific
 		// minimal resource version, simply forward the request to storage.
 		return c.storage.GetList(ctx, key, opts, listObj)
 	}
+	if listRV == 0 && utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache) {
+		listRV, err = c.getCurrentResourceVersionFromStorage(ctx)
+		if err != nil {
+			return err
+		}
+	}
 
 	ctx, span := tracing.Start(ctx, "cacher list",
 		attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
@@ -795,24 +808,30 @@ func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptio
 		return err
 	}
 	span.AddEvent("Listed items from cache", attribute.Int("count", len(objs)))
-	if len(objs) > listVal.Cap() && pred.Label.Empty() && pred.Field.Empty() {
-		// Resize the slice appropriately, since we already know that none
-		// of the elements will be filtered out.
-		listVal.Set(reflect.MakeSlice(reflect.SliceOf(c.objectType.Elem()), 0, len(objs)))
-		span.AddEvent("Resized result")
-	}
+	// store pointer of eligible objects,
+	// Why not directly put object in the items of listObj?
+	//   the elements in ListObject are Struct type, making slice will bring excessive memory consumption.
+	//   so we try to delay this action as much as possible
+	var selectedObjects []runtime.Object
 	for _, obj := range objs {
 		elem, ok := obj.(*storeElement)
 		if !ok {
 			return fmt.Errorf("non *storeElement returned from storage: %v", obj)
 		}
 		if filter(elem.Key, elem.Labels, elem.Fields) {
-			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
+			selectedObjects = append(selectedObjects, elem.Object)
 		}
 	}
-	if listVal.IsNil() {
+	if len(selectedObjects) == 0 {
 		// Ensure that we never return a nil Items pointer in the result for consistency.
 		listVal.Set(reflect.MakeSlice(listVal.Type(), 0, 0))
+	} else {
+		// Resize the slice appropriately, since we already know that size of result set
+		listVal.Set(reflect.MakeSlice(listVal.Type(), len(selectedObjects), len(selectedObjects)))
+		span.AddEvent("Resized result")
+		for i, o := range selectedObjects {
+			listVal.Index(i).Set(reflect.ValueOf(o).Elem())
+		}
 	}
 	span.AddEvent("Filtered items", attribute.Int("count", listVal.Len()))
 	if c.versioner != nil {
@@ -911,9 +930,25 @@ func (c *Cacher) dispatchEvents() {
 			bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25))
 			// Never send a bookmark event if we did not see an event here, this is fine
 			// because we don't provide any guarantees on sending bookmarks.
+			//
+			// Just pop closed watchers and requeue others if needed.
+			//
+			// TODO(#115478): rework the following logic
+			//  in a way that would allow more
+			//  efficient cleanup of closed watchers
 			if lastProcessedResourceVersion == 0 {
-				// pop expired watchers in case there has been no update
-				c.bookmarkWatchers.popExpiredWatchers()
+				func() {
+					c.Lock()
+					defer c.Unlock()
+					for _, watchers := range c.bookmarkWatchers.popExpiredWatchersThreadUnsafe() {
+						for _, watcher := range watchers {
+							if watcher.stopped {
+								continue
+							}
+							c.bookmarkWatchers.addWatcherThreadUnsafe(watcher)
+						}
+					}
+				}()
 				continue
 			}
 			bookmarkEvent := &watchCacheEvent{
@@ -1035,7 +1070,7 @@ func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
 func (c *Cacher) startDispatchingBookmarkEventsLocked() {
 	// Pop already expired watchers. However, explicitly ignore stopped ones,
 	// as we don't delete watcher from bookmarkWatchers when it is stopped.
-	for _, watchers := range c.bookmarkWatchers.popExpiredWatchers() {
+	for _, watchers := range c.bookmarkWatchers.popExpiredWatchersThreadUnsafe() {
 		for _, watcher := range watchers {
 			// c.Lock() is held here.
 			// watcher.stopThreadUnsafe() is protected by c.Lock()
@@ -1140,7 +1175,7 @@ func (c *Cacher) finishDispatching() {
 			continue
 		}
 		// requeue the watcher for the next bookmark if needed.
-		c.bookmarkWatchers.addWatcher(watcher)
+		c.bookmarkWatchers.addWatcherThreadUnsafe(watcher)
 	}
 	c.expiredBookmarkWatchers = c.expiredBookmarkWatchers[:0]
 }
@@ -1309,54 +1344,6 @@ func (c *Cacher) waitUntilWatchCacheFreshAndForceAllEvents(ctx context.Context,
 	return false, nil
 }
 
-// cacherListerWatcher opaques storage.Interface to expose cache.ListerWatcher.
-type cacherListerWatcher struct {
-	storage        storage.Interface
-	resourcePrefix string
-	newListFunc    func() runtime.Object
-}
-
-// NewCacherListerWatcher returns a storage.Interface backed ListerWatcher.
-func NewCacherListerWatcher(storage storage.Interface, resourcePrefix string, newListFunc func() runtime.Object) cache.ListerWatcher {
-	return &cacherListerWatcher{
-		storage:        storage,
-		resourcePrefix: resourcePrefix,
-		newListFunc:    newListFunc,
-	}
-}
-
-// Implements cache.ListerWatcher interface.
-func (lw *cacherListerWatcher) List(options metav1.ListOptions) (runtime.Object, error) {
-	list := lw.newListFunc()
-	pred := storage.SelectionPredicate{
-		Label:    labels.Everything(),
-		Field:    fields.Everything(),
-		Limit:    options.Limit,
-		Continue: options.Continue,
-	}
-
-	storageOpts := storage.ListOptions{
-		ResourceVersionMatch: options.ResourceVersionMatch,
-		Predicate:            pred,
-		Recursive:            true,
-	}
-	if err := lw.storage.GetList(context.TODO(), lw.resourcePrefix, storageOpts, list); err != nil {
-		return nil, err
-	}
-	return list, nil
-}
-
-// Implements cache.ListerWatcher interface.
-func (lw *cacherListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
-	opts := storage.ListOptions{
-		ResourceVersion: options.ResourceVersion,
-		Predicate:       storage.Everything,
-		Recursive:       true,
-		ProgressNotify:  true,
-	}
-	return lw.storage.Watch(context.TODO(), lw.resourcePrefix, opts)
-}
-
 // errWatcher implements watch.Interface to return a single error
 type errWatcher struct {
 	result chan watch.Event

vendor/k8s.io/apiserver/pkg/storage/cacher/caching_object.go

@@ -148,6 +148,10 @@ func (o *cachingObject) CacheEncode(id runtime.Identifier, encode func(runtime.O
 	if result.err != nil {
 		return result.err
 	}
+	if b, support := w.(runtime.Splice); support {
+		b.Splice(result.raw)
+		return nil
+	}
 	_, err := w.Write(result.raw)
 	return err
 }

vendor/k8s.io/apiserver/pkg/storage/cacher/lister_watcher.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cacher
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/client-go/tools/cache"
+)
+
+// listerWatcher opaques storage.Interface to expose cache.ListerWatcher.
+type listerWatcher struct {
+	storage        storage.Interface
+	resourcePrefix string
+	newListFunc    func() runtime.Object
+}
+
+// NewListerWatcher returns a storage.Interface backed ListerWatcher.
+func NewListerWatcher(storage storage.Interface, resourcePrefix string, newListFunc func() runtime.Object) cache.ListerWatcher {
+	return &listerWatcher{
+		storage:        storage,
+		resourcePrefix: resourcePrefix,
+		newListFunc:    newListFunc,
+	}
+}
+
+// Implements cache.ListerWatcher interface.
+func (lw *listerWatcher) List(options metav1.ListOptions) (runtime.Object, error) {
+	list := lw.newListFunc()
+	pred := storage.SelectionPredicate{
+		Label:    labels.Everything(),
+		Field:    fields.Everything(),
+		Limit:    options.Limit,
+		Continue: options.Continue,
+	}
+
+	storageOpts := storage.ListOptions{
+		ResourceVersionMatch: options.ResourceVersionMatch,
+		Predicate:            pred,
+		Recursive:            true,
+	}
+	if err := lw.storage.GetList(context.TODO(), lw.resourcePrefix, storageOpts, list); err != nil {
+		return nil, err
+	}
+	return list, nil
+}
+
+// Implements cache.ListerWatcher interface.
+func (lw *listerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
+	opts := storage.ListOptions{
+		ResourceVersion: options.ResourceVersion,
+		Predicate:       storage.Everything,
+		Recursive:       true,
+		ProgressNotify:  true,
+	}
+	return lw.storage.Watch(context.TODO(), lw.resourcePrefix, opts)
+}

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -30,8 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/cacher/metrics"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/component-base/tracing"
 	"k8s.io/klog/v2"
@@ -196,6 +198,10 @@ type watchCache struct {
 
 	// For testing cache interval invalidation.
 	indexValidator indexValidator
+
+	// Requests progress notification if there are requests waiting for watch
+	// to be fresh
+	waitingUntilFresh *conditionalProgressRequester
 }
 
 func newWatchCache(
@@ -204,8 +210,9 @@ func newWatchCache(
 	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, error),
 	versioner storage.Versioner,
 	indexers *cache.Indexers,
-	clock clock.Clock,
-	groupResource schema.GroupResource) *watchCache {
+	clock clock.WithTicker,
+	groupResource schema.GroupResource,
+	progressRequester *conditionalProgressRequester) *watchCache {
 	wc := &watchCache{
 		capacity:            defaultLowerBoundCapacity,
 		keyFunc:             keyFunc,
@@ -222,6 +229,7 @@ func newWatchCache(
 		clock:               clock,
 		versioner:           versioner,
 		groupResource:       groupResource,
+		waitingUntilFresh:   progressRequester,
 	}
 	metrics.WatchCacheCapacity.WithLabelValues(groupResource.String()).Set(float64(wc.capacity))
 	wc.cond = sync.NewCond(wc.RLocker())
@@ -305,7 +313,7 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 
 	if err := func() error {
 		// TODO: We should consider moving this lock below after the watchCacheEvent
-		// is created. In such situation, the only problematic scenario is Replace(
+		// is created. In such situation, the only problematic scenario is Replace()
 		// happening after getting object from store and before acquiring a lock.
 		// Maybe introduce another lock for this purpose.
 		w.Lock()
@@ -406,6 +414,7 @@ func (w *watchCache) UpdateResourceVersion(resourceVersion string) {
 		w.Lock()
 		defer w.Unlock()
 		w.resourceVersion = rv
+		w.cond.Broadcast()
 	}()
 
 	// Avoid calling event handler under lock.
@@ -484,7 +493,14 @@ func (s sortableStoreElements) Swap(i, j int) {
 // WaitUntilFreshAndList returns list of pointers to `storeElement` objects along
 // with their ResourceVersion and the name of the index, if any, that was used.
 func (w *watchCache) WaitUntilFreshAndList(ctx context.Context, resourceVersion uint64, matchValues []storage.MatchValue) ([]interface{}, uint64, string, error) {
-	err := w.waitUntilFreshAndBlock(ctx, resourceVersion)
+	var err error
+	if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache) && w.notFresh(resourceVersion) {
+		w.waitingUntilFresh.Add()
+		err = w.waitUntilFreshAndBlock(ctx, resourceVersion)
+		w.waitingUntilFresh.Remove()
+	} else {
+		err = w.waitUntilFreshAndBlock(ctx, resourceVersion)
+	}
 	defer w.RUnlock()
 	if err != nil {
 		return nil, 0, "", err
@@ -507,6 +523,12 @@ func (w *watchCache) WaitUntilFreshAndList(ctx context.Context, resourceVersion
 	return result, rv, index, err
 }
 
+func (w *watchCache) notFresh(resourceVersion uint64) bool {
+	w.RLock()
+	defer w.RUnlock()
+	return resourceVersion > w.resourceVersion
+}
+
 // WaitUntilFreshAndGet returns a pointers to <storeElement> object.
 func (w *watchCache) WaitUntilFreshAndGet(ctx context.Context, resourceVersion uint64, key string) (interface{}, bool, uint64, error) {
 	err := w.waitUntilFreshAndBlock(ctx, resourceVersion)
@@ -608,8 +630,8 @@ func (w *watchCache) Resync() error {
 }
 
 func (w *watchCache) currentCapacity() int {
-	w.Lock()
-	defer w.Unlock()
+	w.RLock()
+	defer w.RUnlock()
 	return w.capacity
 }
 

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_progress.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cacher
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"k8s.io/klog/v2"
+	"k8s.io/utils/clock"
+)
+
+const (
+	// progressRequestPeriod determines period of requesting progress
+	// from etcd when there is a request waiting for watch cache to be fresh.
+	progressRequestPeriod = 100 * time.Millisecond
+)
+
+func newConditionalProgressRequester(requestWatchProgress WatchProgressRequester, clock TickerFactory) *conditionalProgressRequester {
+	pr := &conditionalProgressRequester{
+		clock:                clock,
+		requestWatchProgress: requestWatchProgress,
+	}
+	pr.cond = sync.NewCond(pr.mux.RLocker())
+	return pr
+}
+
+type WatchProgressRequester func(ctx context.Context) error
+
+type TickerFactory interface {
+	NewTicker(time.Duration) clock.Ticker
+}
+
+// conditionalProgressRequester will request progress notification if there
+// is a request waiting for watch cache to be fresh.
+type conditionalProgressRequester struct {
+	clock                TickerFactory
+	requestWatchProgress WatchProgressRequester
+
+	mux     sync.RWMutex
+	cond    *sync.Cond
+	waiting int
+	stopped bool
+}
+
+func (pr *conditionalProgressRequester) Run(stopCh <-chan struct{}) {
+	ctx := wait.ContextForChannel(stopCh)
+	go func() {
+		defer utilruntime.HandleCrash()
+		<-stopCh
+		pr.mux.Lock()
+		defer pr.mux.Unlock()
+		pr.stopped = true
+		pr.cond.Signal()
+	}()
+	ticker := pr.clock.NewTicker(progressRequestPeriod)
+	defer ticker.Stop()
+	for {
+		stopped := func() bool {
+			pr.mux.RLock()
+			defer pr.mux.RUnlock()
+			for pr.waiting == 0 && !pr.stopped {
+				pr.cond.Wait()
+			}
+			return pr.stopped
+		}()
+		if stopped {
+			return
+		}
+
+		select {
+		case <-ticker.C():
+			shouldRequest := func() bool {
+				pr.mux.RLock()
+				defer pr.mux.RUnlock()
+				return pr.waiting > 0 && !pr.stopped
+			}()
+			if !shouldRequest {
+				continue
+			}
+			err := pr.requestWatchProgress(ctx)
+			if err != nil {
+				klog.V(4).InfoS("Error requesting bookmark", "err", err)
+			}
+		case <-stopCh:
+			return
+		}
+	}
+}
+
+func (pr *conditionalProgressRequester) Add() {
+	pr.mux.Lock()
+	defer pr.mux.Unlock()
+	pr.waiting += 1
+	pr.cond.Signal()
+}
+
+func (pr *conditionalProgressRequester) Remove() {
+	pr.mux.Lock()
+	defer pr.mux.Unlock()
+	pr.waiting -= 1
+	pr.cond.Signal()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go

@@ -17,11 +17,14 @@ limitations under the License.
 package metrics
 
 import (
+	"context"
+	"fmt"
 	"sync"
 	"time"
 
 	compbasemetrics "k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
 )
 
 /*
@@ -47,6 +50,22 @@ var (
 		},
 		[]string{"operation", "type"},
 	)
+	etcdRequestCounts = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "etcd_requests_total",
+			Help:           "Etcd request counts for each operation and object type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "type"},
+	)
+	etcdRequestErrorCounts = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "etcd_request_errors_total",
+			Help:           "Etcd failed request counts for each operation and object type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "type"},
+	)
 	objectCounts = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
 			Name:           "apiserver_storage_objects",
@@ -57,13 +76,16 @@ var (
 	)
 	dbTotalSize = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
-			Subsystem:      "apiserver",
-			Name:           "storage_db_total_size_in_bytes",
-			Help:           "Total size of the storage database file physically allocated in bytes.",
-			StabilityLevel: compbasemetrics.ALPHA,
+			Subsystem:         "apiserver",
+			Name:              "storage_db_total_size_in_bytes",
+			Help:              "Total size of the storage database file physically allocated in bytes.",
+			StabilityLevel:    compbasemetrics.ALPHA,
+			DeprecatedVersion: "1.28.0",
 		},
 		[]string{"endpoint"},
 	)
+	storageSizeDescription   = compbasemetrics.NewDesc("apiserver_storage_size_bytes", "Size of the storage database file physically allocated in bytes.", []string{"cluster"}, nil, compbasemetrics.ALPHA, "")
+	storageMonitor           = &monitorCollector{monitorGetter: func() ([]Monitor, error) { return nil, nil }}
 	etcdEventsReceivedCounts = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
 			Subsystem:      "apiserver",
@@ -140,8 +162,11 @@ func Register() {
 	// Register the metrics.
 	registerMetrics.Do(func() {
 		legacyregistry.MustRegister(etcdRequestLatency)
+		legacyregistry.MustRegister(etcdRequestCounts)
+		legacyregistry.MustRegister(etcdRequestErrorCounts)
 		legacyregistry.MustRegister(objectCounts)
 		legacyregistry.MustRegister(dbTotalSize)
+		legacyregistry.CustomMustRegister(storageMonitor)
 		legacyregistry.MustRegister(etcdBookmarkCounts)
 		legacyregistry.MustRegister(etcdLeaseObjectCounts)
 		legacyregistry.MustRegister(listStorageCount)
@@ -157,9 +182,15 @@ func UpdateObjectCount(resourcePrefix string, count int64) {
 	objectCounts.WithLabelValues(resourcePrefix).Set(float64(count))
 }
 
-// RecordEtcdRequestLatency sets the etcd_request_duration_seconds metrics.
-func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
-	etcdRequestLatency.WithLabelValues(verb, resource).Observe(sinceInSeconds(startTime))
+// RecordEtcdRequest updates and sets the etcd_request_duration_seconds,
+// etcd_request_total, etcd_request_errors_total metrics.
+func RecordEtcdRequest(verb, resource string, err error, startTime time.Time) {
+	v := []string{verb, resource}
+	etcdRequestLatency.WithLabelValues(v...).Observe(sinceInSeconds(startTime))
+	etcdRequestCounts.WithLabelValues(v...).Inc()
+	if err != nil {
+		etcdRequestErrorCounts.WithLabelValues(v...).Inc()
+	}
 }
 
 // RecordEtcdEvent updated the etcd_events_received_total metric.
@@ -183,15 +214,23 @@ func Reset() {
 }
 
 // sinceInSeconds gets the time since the specified start in seconds.
-func sinceInSeconds(start time.Time) float64 {
+//
+// This is a variable to facilitate testing.
+var sinceInSeconds = func(start time.Time) float64 {
 	return time.Since(start).Seconds()
 }
 
 // UpdateEtcdDbSize sets the etcd_db_total_size_in_bytes metric.
+// Deprecated: Metric etcd_db_total_size_in_bytes will be replaced with apiserver_storage_size_bytes
 func UpdateEtcdDbSize(ep string, size int64) {
 	dbTotalSize.WithLabelValues(ep).Set(float64(size))
 }
 
+// SetStorageMonitorGetter sets monitor getter to allow monitoring etcd stats.
+func SetStorageMonitorGetter(getter func() ([]Monitor, error)) {
+	storageMonitor.monitorGetter = getter
+}
+
 // UpdateLeaseObjectCount sets the etcd_lease_object_counts metric.
 func UpdateLeaseObjectCount(count int64) {
 	// Currently we only store one previous lease, since all the events have the same ttl.
@@ -206,3 +245,51 @@ func RecordStorageListMetrics(resource string, numFetched, numEvald, numReturned
 	listStorageNumSelectorEvals.WithLabelValues(resource).Add(float64(numEvald))
 	listStorageNumReturned.WithLabelValues(resource).Add(float64(numReturned))
 }
+
+type Monitor interface {
+	Monitor(ctx context.Context) (StorageMetrics, error)
+	Close() error
+}
+
+type StorageMetrics struct {
+	Size int64
+}
+
+type monitorCollector struct {
+	compbasemetrics.BaseStableCollector
+
+	monitorGetter func() ([]Monitor, error)
+}
+
+// DescribeWithStability implements compbasemetrics.StableColletor
+func (c *monitorCollector) DescribeWithStability(ch chan<- *compbasemetrics.Desc) {
+	ch <- storageSizeDescription
+}
+
+// CollectWithStability implements compbasemetrics.StableColletor
+func (c *monitorCollector) CollectWithStability(ch chan<- compbasemetrics.Metric) {
+	monitors, err := c.monitorGetter()
+	if err != nil {
+		return
+	}
+
+	for i, m := range monitors {
+		cluster := fmt.Sprintf("etcd-%d", i)
+
+		klog.V(4).InfoS("Start collecting storage metrics", "cluster", cluster)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		metrics, err := m.Monitor(ctx)
+		cancel()
+		m.Close()
+		if err != nil {
+			klog.InfoS("Failed to get storage metrics", "cluster", cluster, "err", err)
+			continue
+		}
+
+		metric, err := compbasemetrics.NewConstMetric(storageSizeDescription, compbasemetrics.GaugeValue, float64(metrics.Size), cluster)
+		if err != nil {
+			klog.ErrorS(err, "Failed to create metric", "cluster", cluster)
+		}
+		ch <- metric
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -85,6 +85,12 @@ type store struct {
 	leaseManager        *leaseManager
 }
 
+func (s *store) RequestWatchProgress(ctx context.Context) error {
+	// Use watchContext to match ctx metadata provided when creating the watch.
+	// In best case scenario we would use the same context that watch was created, but there is no way access it from watchCache.
+	return s.client.RequestProgress(s.watchContext(ctx))
+}
+
 type objState struct {
 	obj   runtime.Object
 	meta  *storage.ResponseMeta
@@ -136,7 +142,7 @@ func (s *store) Get(ctx context.Context, key string, opts storage.GetOptions, ou
 	}
 	startTime := time.Now()
 	getResp, err := s.client.KV.Get(ctx, preparedKey)
-	metrics.RecordEtcdRequestLatency("get", s.groupResourceString, startTime)
+	metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
 	if err != nil {
 		return err
 	}
@@ -210,7 +216,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
 	).Then(
 		clientv3.OpPut(preparedKey, string(newData), opts...),
 	).Commit()
-	metrics.RecordEtcdRequestLatency("create", s.groupResourceString, startTime)
+	metrics.RecordEtcdRequest("create", s.groupResourceString, err, startTime)
 	if err != nil {
 		span.AddEvent("Txn call failed", attribute.String("err", err.Error()))
 		return err
@@ -255,7 +261,7 @@ func (s *store) conditionalDelete(
 	getCurrentState := func() (*objState, error) {
 		startTime := time.Now()
 		getResp, err := s.client.KV.Get(ctx, key)
-		metrics.RecordEtcdRequestLatency("get", s.groupResourceString, startTime)
+		metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
 		if err != nil {
 			return nil, err
 		}
@@ -337,7 +343,7 @@ func (s *store) conditionalDelete(
 		).Else(
 			clientv3.OpGet(key),
 		).Commit()
-		metrics.RecordEtcdRequestLatency("delete", s.groupResourceString, startTime)
+		metrics.RecordEtcdRequest("delete", s.groupResourceString, err, startTime)
 		if err != nil {
 			return err
 		}
@@ -391,7 +397,7 @@ func (s *store) GuaranteedUpdate(
 	getCurrentState := func() (*objState, error) {
 		startTime := time.Now()
 		getResp, err := s.client.KV.Get(ctx, preparedKey)
-		metrics.RecordEtcdRequestLatency("get", s.groupResourceString, startTime)
+		metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
 		if err != nil {
 			return nil, err
 		}
@@ -512,7 +518,7 @@ func (s *store) GuaranteedUpdate(
 		).Else(
 			clientv3.OpGet(preparedKey),
 		).Commit()
-		metrics.RecordEtcdRequestLatency("update", s.groupResourceString, startTime)
+		metrics.RecordEtcdRequest("update", s.groupResourceString, err, startTime)
 		if err != nil {
 			span.AddEvent("Txn call failed", attribute.String("err", err.Error()))
 			return err
@@ -575,7 +581,7 @@ func (s *store) Count(key string) (int64, error) {
 
 	startTime := time.Now()
 	getResp, err := s.client.KV.Get(context.Background(), preparedKey, clientv3.WithRange(clientv3.GetPrefixRangeEnd(preparedKey)), clientv3.WithCountOnly())
-	metrics.RecordEtcdRequestLatency("listWithCount", preparedKey, startTime)
+	metrics.RecordEtcdRequest("listWithCount", preparedKey, err, startTime)
 	if err != nil {
 		return 0, err
 	}
@@ -720,14 +726,16 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 		numReturn := v.Len()
 		metrics.RecordStorageListMetrics(s.groupResourceString, numFetched, numEvald, numReturn)
 	}()
+
+	metricsOp := "get"
+	if recursive {
+		metricsOp = "list"
+	}
+
 	for {
 		startTime := time.Now()
 		getResp, err = s.client.KV.Get(ctx, preparedKey, options...)
-		if recursive {
-			metrics.RecordEtcdRequestLatency("list", s.groupResourceString, startTime)
-		} else {
-			metrics.RecordEtcdRequestLatency("get", s.groupResourceString, startTime)
-		}
+		metrics.RecordEtcdRequest(metricsOp, s.groupResourceString, err, startTime)
 		if err != nil {
 			return interpretListError(err, len(pred.Continue) > 0, continueKey, keyPrefix)
 		}
@@ -863,8 +871,12 @@ func growSlice(v reflect.Value, maxCapacity int, sizes ...int) {
 }
 
 // Watch implements storage.Interface.Watch.
+// TODO(#115478): In order to graduate the WatchList feature to beta, the etcd3 implementation must/should also support it.
 func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
-	if opts.SendInitialEvents != nil {
+	// it is safe to skip SendInitialEvents if the request is backward compatible
+	// see https://github.com/kubernetes/kubernetes/blob/267eb25e60955fe8e438c6311412e7cf7d028acb/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go#L260
+	compatibility := opts.Predicate.AllowWatchBookmarks == false && (opts.ResourceVersion == "" || opts.ResourceVersion == "0")
+	if opts.SendInitialEvents != nil && !compatibility {
 		return nil, apierrors.NewInvalid(
 			schema.GroupKind{Group: s.groupResource.Group, Kind: s.groupResource.Resource},
 			"",
@@ -879,7 +891,18 @@ func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions)
 	if err != nil {
 		return nil, err
 	}
-	return s.watcher.Watch(ctx, preparedKey, int64(rev), opts.Recursive, opts.ProgressNotify, s.transformer, opts.Predicate)
+	return s.watcher.Watch(s.watchContext(ctx), preparedKey, int64(rev), opts.Recursive, opts.ProgressNotify, s.transformer, opts.Predicate)
+}
+
+func (s *store) watchContext(ctx context.Context) context.Context {
+	// The etcd server waits until it cannot find a leader for 3 election
+	// timeouts to cancel existing streams. 3 is currently a hard coded
+	// constant. The election timeout defaults to 1000ms. If the cluster is
+	// healthy, when the leader is stopped, the leadership transfer should be
+	// smooth. (leader transfers its leadership before stopping). If leader is
+	// hard killed, other servers will take an election timeout to realize
+	// leader lost and start campaign.
+	return clientv3.WithRequireLeader(ctx)
 }
 
 func (s *store) getState(ctx context.Context, getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) {

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -144,15 +144,7 @@ func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, re
 		// The filter doesn't filter out any object.
 		wc.internalPred = storage.Everything
 	}
-
-	// The etcd server waits until it cannot find a leader for 3 election
-	// timeouts to cancel existing streams. 3 is currently a hard coded
-	// constant. The election timeout defaults to 1000ms. If the cluster is
-	// healthy, when the leader is stopped, the leadership transfer should be
-	// smooth. (leader transfers its leadership before stopping). If leader is
-	// hard killed, other servers will take an election timeout to realize
-	// leader lost and start campaign.
-	wc.ctx, wc.cancel = context.WithCancel(clientv3.WithRequireLeader(ctx))
+	wc.ctx, wc.cancel = context.WithCancel(ctx)
 	return wc
 }
 
@@ -223,6 +215,10 @@ func (wc *watchChan) ResultChan() <-chan watch.Event {
 	return wc.resultChan
 }
 
+func (wc *watchChan) RequestWatchProgress() error {
+	return wc.watcher.client.RequestProgress(wc.ctx)
+}
+
 // sync tries to retrieve existing data and send them to process.
 // The revision to watch will be set to the revision in response.
 // All events sent will have isCreated=true

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -236,6 +236,21 @@ type Interface interface {
 
 	// Count returns number of different entries under the key (generally being path prefix).
 	Count(key string) (int64, error)
+
+	// RequestWatchProgress requests the a watch stream progress status be sent in the
+	// watch response stream as soon as possible.
+	// Used for monitor watch progress even if watching resources with no changes.
+	//
+	// If watch is lagging, progress status might:
+	// * be pointing to stale resource version. Use etcd KV request to get linearizable resource version.
+	// * not be delivered at all. It's recommended to poll request progress periodically.
+	//
+	// Note: Only watches with matching context grpc metadata will be notified.
+	// https://github.com/kubernetes/kubernetes/blob/9325a57125e8502941d1b0c7379c4bb80a678d5c/vendor/go.etcd.io/etcd/client/v3/watch.go#L1037-L1042
+	//
+	// TODO: Remove when storage.Interface will be separate from etc3.store.
+	// Deprecated: Added temporarily to simplify exposing RequestProgress for watch cache.
+	RequestWatchProgress(ctx context.Context) error
 }
 
 // GetOptions provides the options that may be provided for storage get operations.

vendor/k8s.io/apiserver/pkg/storage/storagebackend/OWNERS

@@ -1,6 +1,5 @@
 # See the OWNERS docs at https://go.k8s.io/owners
 
 reviewers:
-  - lavalamp
   - smarterclayton
   - wojtek-t

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -37,6 +38,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"golang.org/x/time/rate"
 	"google.golang.org/grpc"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
@@ -52,7 +54,6 @@ import (
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/component-base/metrics/legacyregistry"
 	tracing "k8s.io/component-base/tracing"
-	"k8s.io/klog/v2"
 )
 
 const (
@@ -153,11 +154,11 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan
 	// retry in a loop in the background until we successfully create the client, storing the client or error encountered
 
 	lock := sync.RWMutex{}
-	var prober *etcd3Prober
+	var prober *etcd3ProberMonitor
 	clientErr := fmt.Errorf("etcd client connection not yet established")
 
 	go wait.PollUntil(time.Second, func() (bool, error) {
-		newProber, err := newETCD3Prober(c)
+		newProber, err := newETCD3ProberMonitor(c)
 		lock.Lock()
 		defer lock.Unlock()
 		// Ensure that server is already not shutting down.
@@ -221,49 +222,66 @@ func newETCD3Check(c storagebackend.Config, timeout time.Duration, stopCh <-chan
 	}, nil
 }
 
-func newETCD3Prober(c storagebackend.Config) (*etcd3Prober, error) {
+func newETCD3ProberMonitor(c storagebackend.Config) (*etcd3ProberMonitor, error) {
 	client, err := newETCD3Client(c.Transport)
 	if err != nil {
 		return nil, err
 	}
-	return &etcd3Prober{
-		client: client,
-		prefix: c.Prefix,
+	return &etcd3ProberMonitor{
+		client:    client,
+		prefix:    c.Prefix,
+		endpoints: c.Transport.ServerList,
 	}, nil
 }
 
-type etcd3Prober struct {
-	prefix string
+type etcd3ProberMonitor struct {
+	prefix    string
+	endpoints []string
 
 	mux    sync.RWMutex
 	client *clientv3.Client
 	closed bool
 }
 
-func (p *etcd3Prober) Close() error {
-	p.mux.Lock()
-	defer p.mux.Unlock()
-	if !p.closed {
-		p.closed = true
-		return p.client.Close()
+func (t *etcd3ProberMonitor) Close() error {
+	t.mux.Lock()
+	defer t.mux.Unlock()
+	if !t.closed {
+		t.closed = true
+		return t.client.Close()
 	}
-	return fmt.Errorf("prober was closed")
+	return fmt.Errorf("closed")
 }
 
-func (p *etcd3Prober) Probe(ctx context.Context) error {
-	p.mux.RLock()
-	defer p.mux.RUnlock()
-	if p.closed {
-		return fmt.Errorf("prober was closed")
+func (t *etcd3ProberMonitor) Probe(ctx context.Context) error {
+	t.mux.RLock()
+	defer t.mux.RUnlock()
+	if t.closed {
+		return fmt.Errorf("closed")
 	}
 	// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
-	_, err := p.client.Get(ctx, path.Join("/", p.prefix, "health"))
+	_, err := t.client.Get(ctx, path.Join("/", t.prefix, "health"))
 	if err != nil {
 		return fmt.Errorf("error getting data from etcd: %w", err)
 	}
 	return nil
 }
 
+func (t *etcd3ProberMonitor) Monitor(ctx context.Context) (metrics.StorageMetrics, error) {
+	t.mux.RLock()
+	defer t.mux.RUnlock()
+	if t.closed {
+		return metrics.StorageMetrics{}, fmt.Errorf("closed")
+	}
+	status, err := t.client.Status(ctx, t.endpoints[rand.Int()%len(t.endpoints)])
+	if err != nil {
+		return metrics.StorageMetrics{}, err
+	}
+	return metrics.StorageMetrics{
+		Size: status.DbSize,
+	}, nil
+}
+
 var newETCD3Client = func(c storagebackend.TransportConfig) (*clientv3.Client, error) {
 	tlsInfo := transport.TLSInfo{
 		CertFile:      c.CertFile,
@@ -441,6 +459,7 @@ func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.
 
 // startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the
 // corresponding metric etcd_db_total_size_in_bytes for each etcd server endpoint.
+// Deprecated: Will be replaced with newETCD3ProberMonitor
 func startDBSizeMonitorPerEndpoint(client *clientv3.Client, interval time.Duration) (func(), error) {
 	if interval == 0 {
 		return func() {}, nil

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go

@@ -22,6 +22,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 )
 
@@ -68,7 +69,18 @@ func CreateProber(c storagebackend.Config) (Prober, error) {
 	case storagebackend.StorageTypeETCD2:
 		return nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type)
 	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
-		return newETCD3Prober(c)
+		return newETCD3ProberMonitor(c)
+	default:
+		return nil, fmt.Errorf("unknown storage type: %s", c.Type)
+	}
+}
+
+func CreateMonitor(c storagebackend.Config) (metrics.Monitor, error) {
+	switch c.Type {
+	case storagebackend.StorageTypeETCD2:
+		return nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type)
+	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
+		return newETCD3ProberMonitor(c)
 	default:
 		return nil, fmt.Errorf("unknown storage type: %s", c.Type)
 	}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go

@@ -34,33 +34,11 @@ import (
 	"k8s.io/klog/v2"
 )
 
-type gcm struct {
-	aead      cipher.AEAD
-	nonceFunc func([]byte) error
-}
+// commonSize is the length of various security sensitive byte slices such as encryption keys.
+// Do not change this value.  It would be a backward incompatible change.
+const commonSize = 32
 
-// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given data.
-// It implements AEAD encryption of the provided values given a cipher.Block algorithm.
-// The authenticated data provided as part of the value.Context method must match when the same
-// value is set to and loaded from storage. In order to ensure that values cannot be copied by
-// an attacker from a location under their control, use characteristics of the storage location
-// (such as the etcd key) as part of the authenticated data.
-//
-// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
-// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
-// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
-// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
-// therefore transformers using this implementation *must* ensure they allow for frequent key
-// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
-// random nonces.
-func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
-	aead, err := newGCM(block)
-	if err != nil {
-		return nil, err
-	}
-
-	return &gcm{aead: aead, nonceFunc: randomNonce}, nil
-}
+const keySizeCounterNonceGCM = commonSize
 
 // NewGCMTransformerWithUniqueKeyUnsafe is the same as NewGCMTransformer but is unsafe for general
 // use because it makes assumptions about the key underlying the block cipher.  Specifically,
@@ -78,7 +56,7 @@ func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
 // it can be passed to NewGCMTransformer(aes.NewCipher(key)) to construct a transformer capable
 // of decrypting values encrypted by this transformer (that transformer must not be used for encryption).
 func NewGCMTransformerWithUniqueKeyUnsafe() (value.Transformer, []byte, error) {
-	key, err := generateKey(32)
+	key, err := GenerateKey(keySizeCounterNonceGCM)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -126,17 +104,6 @@ func newGCMTransformerWithUniqueKeyUnsafe(block cipher.Block, nonceGen *nonceGen
 	return &gcm{aead: aead, nonceFunc: nonceFunc}, nil
 }
 
-func newGCM(block cipher.Block) (cipher.AEAD, error) {
-	aead, err := cipher.NewGCM(block)
-	if err != nil {
-		return nil, err
-	}
-	if nonceSize := aead.NonceSize(); nonceSize != 12 { // all data in etcd will be broken if this ever changes
-		return nil, fmt.Errorf("crypto/cipher.NewGCM returned unexpected nonce size: %d", nonceSize)
-	}
-	return aead, nil
-}
-
 func randomNonce(b []byte) error {
 	_, err := rand.Read(b)
 	return err
@@ -164,8 +131,8 @@ func die(msg string) {
 	klog.FatalDepth(1, msg)
 }
 
-// generateKey generates a random key using system randomness.
-func generateKey(length int) (key []byte, err error) {
+// GenerateKey generates a random key using system randomness.
+func GenerateKey(length int) (key []byte, err error) {
 	defer func(start time.Time) {
 		value.RecordDataKeyGeneration(start, err)
 	}(time.Now())
@@ -177,6 +144,45 @@ func generateKey(length int) (key []byte, err error) {
 	return key, nil
 }
 
+// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given data.
+// It implements AEAD encryption of the provided values given a cipher.Block algorithm.
+// The authenticated data provided as part of the value.Context method must match when the same
+// value is set to and loaded from storage. In order to ensure that values cannot be copied by
+// an attacker from a location under their control, use characteristics of the storage location
+// (such as the etcd key) as part of the authenticated data.
+//
+// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
+// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
+// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
+// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
+// therefore transformers using this implementation *must* ensure they allow for frequent key
+// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
+// random nonces.
+func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
+	aead, err := newGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	return &gcm{aead: aead, nonceFunc: randomNonce}, nil
+}
+
+func newGCM(block cipher.Block) (cipher.AEAD, error) {
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	if nonceSize := aead.NonceSize(); nonceSize != 12 { // all data in etcd will be broken if this ever changes
+		return nil, fmt.Errorf("crypto/cipher.NewGCM returned unexpected nonce size: %d", nonceSize)
+	}
+	return aead, nil
+}
+
+type gcm struct {
+	aead      cipher.AEAD
+	nonceFunc func([]byte) error
+}
+
 func (t *gcm) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
 	nonceSize := t.aead.NonceSize()
 	if len(data) < nonceSize {

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes_extended_nonce.go

@@ -0,0 +1,186 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aes
+
+import (
+	"bytes"
+	"context"
+	"crypto/aes"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"time"
+
+	"golang.org/x/crypto/hkdf"
+
+	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/utils/clock"
+)
+
+const (
+	// cacheTTL is the TTL of KDF cache entries.  We assume that the value.Context.AuthenticatedData
+	// for every call is the etcd storage path of the associated resource, and use that as the primary
+	// cache key (with a secondary check that confirms that the info matches).  Thus if a client
+	// is constantly creating resources with new names (and thus new paths), they will keep adding new
+	// entries to the cache for up to this TTL before the GC logic starts deleting old entries.  Each
+	// entry is ~300 bytes in size, so even a malicious client will be bounded in the overall memory
+	// it can consume.
+	cacheTTL = 10 * time.Minute
+
+	derivedKeySizeExtendedNonceGCM = commonSize
+	infoSizeExtendedNonceGCM
+	MinSeedSizeExtendedNonceGCM
+)
+
+// NewHKDFExtendedNonceGCMTransformer is the same as NewGCMTransformer but trades storage,
+// memory and CPU to work around the limitations of AES-GCM's 12 byte nonce size.  The input seed
+// is assumed to be a cryptographically strong slice of MinSeedSizeExtendedNonceGCM+ random bytes.
+// Unlike NewGCMTransformer, this function is immune to the birthday attack because a new key is generated
+// per encryption via a key derivation function: KDF(seed, random_bytes) -> key.  The derived key is
+// only used once as an AES-GCM key with a random 12 byte nonce.  This avoids any concerns around
+// cryptographic wear out (by either number of encryptions or the amount of data being encrypted).
+// Speaking on the cryptographic safety, the limit on the number of operations that can be preformed
+// with a single seed with derived keys and randomly generated nonces is not practically reachable.
+// Thus, the scheme does not impose any specific requirements on the seed rotation schedule.
+// Reusing the same seed is safe to do over time and across process restarts.  Whenever a new
+// seed is needed, the caller should generate it via GenerateKey(MinSeedSizeExtendedNonceGCM).
+// In regard to KMSv2, organization standards or compliance policies around rotation may require
+// that the seed be rotated at some interval.  This can be implemented externally by rotating
+// the key encryption key via a key ID change.
+func NewHKDFExtendedNonceGCMTransformer(seed []byte) (value.Transformer, error) {
+	if seedLen := len(seed); seedLen < MinSeedSizeExtendedNonceGCM {
+		return nil, fmt.Errorf("invalid seed length %d used for key generation", seedLen)
+	}
+	return &extendedNonceGCM{
+		seed:  seed,
+		cache: newSimpleCache(clock.RealClock{}, cacheTTL),
+	}, nil
+}
+
+type extendedNonceGCM struct {
+	seed  []byte
+	cache *simpleCache
+}
+
+func (e *extendedNonceGCM) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
+	if len(data) < infoSizeExtendedNonceGCM {
+		return nil, false, errors.New("the stored data was shorter than the required size")
+	}
+
+	info := data[:infoSizeExtendedNonceGCM]
+
+	transformer, err := e.derivedKeyTransformer(info, dataCtx, false)
+	if err != nil {
+		return nil, false, fmt.Errorf("failed to derive read key from KDF: %w", err)
+	}
+
+	return transformer.TransformFromStorage(ctx, data, dataCtx)
+}
+
+func (e *extendedNonceGCM) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
+	info := make([]byte, infoSizeExtendedNonceGCM)
+	if err := randomNonce(info); err != nil {
+		return nil, fmt.Errorf("failed to generate info for KDF: %w", err)
+	}
+
+	transformer, err := e.derivedKeyTransformer(info, dataCtx, true)
+	if err != nil {
+		return nil, fmt.Errorf("failed to derive write key from KDF: %w", err)
+	}
+
+	return transformer.TransformToStorage(ctx, data, dataCtx)
+}
+
+func (e *extendedNonceGCM) derivedKeyTransformer(info []byte, dataCtx value.Context, write bool) (value.Transformer, error) {
+	if !write { // no need to check cache on write since we always generate a new transformer
+		if transformer := e.cache.get(info, dataCtx); transformer != nil {
+			return transformer, nil
+		}
+
+		// on read, this is a subslice of a much larger slice and we do not want to hold onto that larger slice
+		info = bytes.Clone(info)
+	}
+
+	key, err := e.sha256KDFExpandOnly(info)
+	if err != nil {
+		return nil, fmt.Errorf("failed to KDF expand seed with info: %w", err)
+	}
+
+	transformer, err := newGCMTransformerWithInfo(key, info)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build transformer with KDF derived key: %w", err)
+	}
+
+	e.cache.set(dataCtx, transformer)
+
+	return transformer, nil
+}
+
+func (e *extendedNonceGCM) sha256KDFExpandOnly(info []byte) ([]byte, error) {
+	kdf := hkdf.Expand(sha256.New, e.seed, info)
+
+	derivedKey := make([]byte, derivedKeySizeExtendedNonceGCM)
+	if _, err := io.ReadFull(kdf, derivedKey); err != nil {
+		return nil, fmt.Errorf("failed to read a derived key from KDF: %w", err)
+	}
+
+	return derivedKey, nil
+}
+
+func newGCMTransformerWithInfo(key, info []byte) (*transformerWithInfo, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+
+	transformer, err := NewGCMTransformer(block)
+	if err != nil {
+		return nil, err
+	}
+
+	return &transformerWithInfo{transformer: transformer, info: info}, nil
+}
+
+type transformerWithInfo struct {
+	transformer value.Transformer
+	// info are extra opaque bytes prepended to the writes from transformer and stripped from reads.
+	// currently info is used to generate a key via KDF(seed, info) -> key
+	// and transformer is the output of NewGCMTransformer(aes.NewCipher(key))
+	info []byte
+}
+
+func (t *transformerWithInfo) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
+	if !bytes.HasPrefix(data, t.info) {
+		return nil, false, errors.New("the stored data is missing the required info prefix")
+	}
+
+	return t.transformer.TransformFromStorage(ctx, data[len(t.info):], dataCtx)
+}
+
+func (t *transformerWithInfo) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
+	out, err := t.transformer.TransformToStorage(ctx, data, dataCtx)
+	if err != nil {
+		return nil, err
+	}
+
+	outWithInfo := make([]byte, 0, len(out)+len(t.info))
+	outWithInfo = append(outWithInfo, t.info...)
+	outWithInfo = append(outWithInfo, out...)
+
+	return outWithInfo, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/cache.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package aes
+
+import (
+	"bytes"
+	"time"
+	"unsafe"
+
+	utilcache "k8s.io/apimachinery/pkg/util/cache"
+	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/utils/clock"
+)
+
+type simpleCache struct {
+	cache *utilcache.Expiring
+	ttl   time.Duration
+}
+
+func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
+	cache := utilcache.NewExpiringWithClock(clock)
+	// "Stale" entries are always valid for us because the TTL is just used to prevent
+	// unbounded growth on the cache - for a given info the transformer is always the same.
+	// The key always corresponds to the exact same value, with the caveat that
+	// since we use the value.Context.AuthenticatedData to overwrite old keys,
+	// we always have to check that the info matches (to validate the transformer is correct).
+	cache.AllowExpiredGet = true
+	return &simpleCache{
+		cache: cache,
+		ttl:   ttl,
+	}
+}
+
+// given a key, return the transformer, or nil if it does not exist in the cache
+func (c *simpleCache) get(info []byte, dataCtx value.Context) *transformerWithInfo {
+	val, ok := c.cache.Get(keyFunc(dataCtx))
+	if !ok {
+		return nil
+	}
+
+	transformer := val.(*transformerWithInfo)
+
+	if !bytes.Equal(transformer.info, info) {
+		return nil
+	}
+
+	return transformer
+}
+
+// set caches the record for the key
+func (c *simpleCache) set(dataCtx value.Context, transformer *transformerWithInfo) {
+	if dataCtx == nil || len(dataCtx.AuthenticatedData()) == 0 {
+		panic("authenticated data must not be empty")
+	}
+	if transformer == nil {
+		panic("transformer must not be nil")
+	}
+	if len(transformer.info) == 0 {
+		panic("info must not be empty")
+	}
+	c.cache.Set(keyFunc(dataCtx), transformer, c.ttl)
+}
+
+func keyFunc(dataCtx value.Context) string {
+	return toString(dataCtx.AuthenticatedData())
+}
+
+// toString performs unholy acts to avoid allocations
+func toString(b []byte) string {
+	// unsafe.SliceData relies on cap whereas we want to rely on len
+	if len(b) == 0 {
+		return ""
+	}
+	// Copied from go 1.20.1 strings.Builder.String
+	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/strings/builder.go#L48
+	return unsafe.String(unsafe.SliceData(b), len(b))
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/cache.go

@@ -18,7 +18,6 @@ limitations under the License.
 package kmsv2
 
 import (
-	"context"
 	"crypto/sha256"
 	"hash"
 	"sync"
@@ -30,17 +29,10 @@ import (
 	"k8s.io/utils/clock"
 )
 
-// prevent decryptTransformer from drifting from value.Transformer
-var _ decryptTransformer = value.Transformer(nil)
-
-// decryptTransformer is the decryption subset of value.Transformer.
-// this exists purely to statically enforce that transformers placed in the cache are not used for encryption.
+// simpleCache stores the decryption subset of value.Transformer (value.Read).
+// this statically enforces that transformers placed in the cache are not used for encryption.
 // this is relevant in the context of nonce collision since transformers that are created
 // from encrypted DEKs retrieved from etcd cannot maintain their nonce counter state.
-type decryptTransformer interface {
-	TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error)
-}
-
 type simpleCache struct {
 	cache *utilcache.Expiring
 	ttl   time.Duration
@@ -50,8 +42,10 @@ type simpleCache struct {
 }
 
 func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
+	cache := utilcache.NewExpiringWithClock(clock)
+	cache.AllowExpiredGet = true // for a given key, the value (the decryptTransformer) is always the same
 	return &simpleCache{
-		cache: utilcache.NewExpiringWithClock(clock),
+		cache: cache,
 		ttl:   ttl,
 		hashPool: &sync.Pool{
 			New: func() interface{} {
@@ -62,16 +56,16 @@ func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
 }
 
 // given a key, return the transformer, or nil if it does not exist in the cache
-func (c *simpleCache) get(key []byte) decryptTransformer {
+func (c *simpleCache) get(key []byte) value.Read {
 	record, ok := c.cache.Get(c.keyFunc(key))
 	if !ok {
 		return nil
 	}
-	return record.(decryptTransformer)
+	return record.(value.Read)
 }
 
 // set caches the record for the key
-func (c *simpleCache) set(key []byte, transformer decryptTransformer) {
+func (c *simpleCache) set(key []byte, transformer value.Read) {
 	if len(key) == 0 {
 		panic("key must not be empty")
 	}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -20,6 +20,8 @@ package kmsv2
 import (
 	"context"
 	"crypto/aes"
+	"crypto/cipher"
+	"crypto/sha256"
 	"fmt"
 	"sort"
 	"time"
@@ -42,6 +44,8 @@ import (
 	"k8s.io/utils/clock"
 )
 
+// TODO integration test with old AES GCM data recorded and new KDF data recorded
+
 func init() {
 	value.RegisterMetrics()
 	metrics.RegisterMetrics()
@@ -54,22 +58,22 @@ const (
 	annotationsMaxSize = 32 * 1024 // 32 kB
 	// KeyIDMaxSize is the maximum size of the keyID.
 	KeyIDMaxSize = 1 * 1024 // 1 kB
-	// encryptedDEKMaxSize is the maximum size of the encrypted DEK.
-	encryptedDEKMaxSize = 1 * 1024 // 1 kB
+	// encryptedDEKSourceMaxSize is the maximum size of the encrypted DEK source.
+	encryptedDEKSourceMaxSize = 1 * 1024 // 1 kB
 	// cacheTTL is the default time-to-live for the cache entry.
 	// this allows the cache to grow to an infinite size for up to a day.
-	// this is meant as a temporary solution until the cache is re-written to not have a TTL.
 	// there is unlikely to be any meaningful memory impact on the server
-	// because the cache will likely never have more than a few thousand entries
-	// and each entry is roughly ~200 bytes in size.  with DEK reuse
-	// and no storage migration, the number of entries in this cache
+	// because the cache will likely never have more than a few thousand entries.
+	// each entry can be large due to an internal cache that maps the DEK seed to individual
+	// DEK entries, but that cache has an aggressive TTL to keep the size under control.
+	// with DEK/seed reuse and no storage migration, the number of entries in this cache
 	// would be approximated by unique key IDs used by the KMS plugin
 	// combined with the number of server restarts.  If storage migration
 	// is performed after key ID changes, and the number of restarts
 	// is limited, this cache size may be as small as the number of API
 	// servers in use (once old entries expire out from the TTL).
 	cacheTTL = 24 * time.Hour
-	// error code
+	// key ID related error codes for metrics
 	errKeyIDOKCode      ErrCodeKeyID = "ok"
 	errKeyIDEmptyCode   ErrCodeKeyID = "empty"
 	errKeyIDTooLongCode ErrCodeKeyID = "too_long"
@@ -82,23 +86,22 @@ type StateFunc func() (State, error)
 type ErrCodeKeyID string
 
 type State struct {
-	Transformer  value.Transformer
-	EncryptedDEK []byte
-	KeyID        string
-	Annotations  map[string][]byte
+	Transformer value.Transformer
+
+	EncryptedObject kmstypes.EncryptedObject
 
 	UID string
 
 	ExpirationTimestamp time.Time
 
-	// CacheKey is the key used to cache the DEK in transformer.cache.
+	// CacheKey is the key used to cache the DEK/seed in envelopeTransformer.cache.
 	CacheKey []byte
 }
 
 func (s *State) ValidateEncryptCapability() error {
 	if now := NowFunc(); now.After(s.ExpirationTimestamp) {
-		return fmt.Errorf("EDEK with keyID %q expired at %s (current time is %s)",
-			s.KeyID, s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
+		return fmt.Errorf("encryptedDEKSource with keyID hash %q expired at %s (current time is %s)",
+			GetHashIfNotEmpty(s.EncryptedObject.KeyID), s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
 	}
 	return nil
 }
@@ -136,6 +139,8 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 		return nil, false, err
 	}
 
+	useSeed := encryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
+
 	// TODO: consider marking state.EncryptedDEK != encryptedObject.EncryptedDEK as a stale read to support DEK defragmentation
 	//  at a minimum we should have a metric that helps the user understand if DEK fragmentation is high
 	state, err := t.stateFunc() // no need to call state.ValidateEncryptCapability on reads
@@ -143,7 +148,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 		return nil, false, err
 	}
 
-	encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEK, encryptedObject.KeyID, encryptedObject.Annotations)
+	encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEKSourceType, encryptedObject.EncryptedDEKSource, encryptedObject.KeyID, encryptedObject.Annotations)
 	if err != nil {
 		return nil, false, err
 	}
@@ -162,7 +167,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 			"verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
 
 		key, err := t.envelopeService.Decrypt(ctx, uid, &kmsservice.DecryptRequest{
-			Ciphertext:  encryptedObject.EncryptedDEK,
+			Ciphertext:  encryptedObject.EncryptedDEKSource,
 			KeyID:       encryptedObject.KeyID,
 			Annotations: encryptedObject.Annotations,
 		})
@@ -170,7 +175,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 			return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
 		}
 
-		transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key)
+		transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key, useSeed)
 		if err != nil {
 			return nil, false, err
 		}
@@ -183,8 +188,11 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 	}
 
 	// data is considered stale if the key ID does not match our current write transformer
-	return out, stale || encryptedObject.KeyID != state.KeyID, nil
-
+	return out,
+		stale ||
+			encryptedObject.KeyID != state.EncryptedObject.KeyID ||
+			encryptedObject.EncryptedDEKSourceType != state.EncryptedObject.EncryptedDEKSourceType,
+		nil
 }
 
 // TransformToStorage encrypts data to be written to disk using envelope encryption.
@@ -200,7 +208,7 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 	// this prevents a cache miss every time the DEK rotates
 	// this has the side benefit of causing the cache to perform a GC
 	// TODO see if we can do this inside the stateFunc control loop
-	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
+	// TODO(aramase): Add metrics for cache size.
 	t.cache.set(state.CacheKey, state.Transformer)
 
 	requestInfo := getRequestInfoFromContext(ctx)
@@ -213,39 +221,43 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 		return nil, err
 	}
 
-	metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.KeyID)
+	metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID)
 
-	encObject := &kmstypes.EncryptedObject{
-		KeyID:         state.KeyID,
-		EncryptedDEK:  state.EncryptedDEK,
-		EncryptedData: result,
-		Annotations:   state.Annotations,
-	}
+	encObjectCopy := state.EncryptedObject
+	encObjectCopy.EncryptedData = result
 
 	// Serialize the EncryptedObject to a byte array.
-	return t.doEncode(encObject)
+	return t.doEncode(&encObjectCopy)
 }
 
 // addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
-func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte) (decryptTransformer, error) {
-	block, err := aes.NewCipher(key)
+func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte, useSeed bool) (value.Read, error) {
+	var transformer value.Read
+	var err error
+	if useSeed {
+		// the input key is considered safe to use here because it is coming from the KMS plugin / etcd
+		transformer, err = aestransformer.NewHKDFExtendedNonceGCMTransformer(key)
+	} else {
+		var block cipher.Block
+		block, err = aes.NewCipher(key)
+		if err != nil {
+			return nil, err
+		}
+		// this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
+		// it would use random nonces for encryption but we never do that
+		transformer, err = aestransformer.NewGCMTransformer(block)
+	}
 	if err != nil {
 		return nil, err
 	}
-	// this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
-	// it would use random nonces for encryption but we never do that
-	transformer, err := aestransformer.NewGCMTransformer(block)
-	if err != nil {
-		return nil, err
-	}
-	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
+	// TODO(aramase): Add metrics for cache size.
 	t.cache.set(cacheKey, transformer)
 	return transformer, nil
 }
 
 // doEncode encodes the EncryptedObject to a byte array.
 func (t *envelopeTransformer) doEncode(request *kmstypes.EncryptedObject) ([]byte, error) {
-	if err := validateEncryptedObject(request); err != nil {
+	if err := ValidateEncryptedObject(request); err != nil {
 		return nil, err
 	}
 	return proto.Marshal(request)
@@ -257,16 +269,31 @@ func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.Encrypted
 	if err := proto.Unmarshal(originalData, o); err != nil {
 		return nil, err
 	}
-	// validate the EncryptedObject
-	if err := validateEncryptedObject(o); err != nil {
+	if err := ValidateEncryptedObject(o); err != nil {
 		return nil, err
 	}
 
 	return o, nil
 }
 
-func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, []byte, error) {
-	transformer, newKey, err := aestransformer.NewGCMTransformerWithUniqueKeyUnsafe()
+// GenerateTransformer generates a new transformer and encrypts the DEK/seed using the envelope service.
+// It returns the transformer, the encrypted DEK/seed, cache key and error.
+func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service, useSeed bool) (value.Transformer, *kmstypes.EncryptedObject, []byte, error) {
+	newTransformerFunc := func() (value.Transformer, []byte, error) {
+		seed, err := aestransformer.GenerateKey(aestransformer.MinSeedSizeExtendedNonceGCM)
+		if err != nil {
+			return nil, nil, err
+		}
+		transformer, err := aestransformer.NewHKDFExtendedNonceGCMTransformer(seed)
+		if err != nil {
+			return nil, nil, err
+		}
+		return transformer, seed, nil
+	}
+	if !useSeed {
+		newTransformerFunc = aestransformer.NewGCMTransformerWithUniqueKeyUnsafe
+	}
+	transformer, newKey, err := newTransformerFunc()
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -278,32 +305,48 @@ func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsser
 		return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
 	}
 
-	if err := validateEncryptedObject(&kmstypes.EncryptedObject{
-		KeyID:         resp.KeyID,
-		EncryptedDEK:  resp.Ciphertext,
-		EncryptedData: []byte{0}, // any non-empty value to pass validation
-		Annotations:   resp.Annotations,
-	}); err != nil {
+	o := &kmstypes.EncryptedObject{
+		KeyID:              resp.KeyID,
+		EncryptedDEKSource: resp.Ciphertext,
+		EncryptedData:      []byte{0}, // any non-empty value to pass validation
+		Annotations:        resp.Annotations,
+	}
+
+	if useSeed {
+		o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
+	} else {
+		o.EncryptedDEKSourceType = kmstypes.EncryptedDEKSourceType_AES_GCM_KEY
+	}
+
+	if err := ValidateEncryptedObject(o); err != nil {
 		return nil, nil, nil, err
 	}
 
-	cacheKey, err := generateCacheKey(resp.Ciphertext, resp.KeyID, resp.Annotations)
+	cacheKey, err := generateCacheKey(o.EncryptedDEKSourceType, resp.Ciphertext, resp.KeyID, resp.Annotations)
 	if err != nil {
 		return nil, nil, nil, err
 	}
 
-	return transformer, resp, cacheKey, nil
+	o.EncryptedData = nil // make sure that later code that uses this encrypted object sets this field
+
+	return transformer, o, cacheKey, nil
 }
 
-func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
+func ValidateEncryptedObject(o *kmstypes.EncryptedObject) error {
 	if o == nil {
 		return fmt.Errorf("encrypted object is nil")
 	}
+	switch t := o.EncryptedDEKSourceType; t {
+	case kmstypes.EncryptedDEKSourceType_AES_GCM_KEY:
+	case kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED:
+	default:
+		return fmt.Errorf("unknown encryptedDEKSourceType: %d", t)
+	}
 	if len(o.EncryptedData) == 0 {
 		return fmt.Errorf("encrypted data is empty")
 	}
-	if err := validateEncryptedDEK(o.EncryptedDEK); err != nil {
-		return fmt.Errorf("failed to validate encrypted DEK: %w", err)
+	if err := validateEncryptedDEKSource(o.EncryptedDEKSource); err != nil {
+		return fmt.Errorf("failed to validate encrypted DEK source: %w", err)
 	}
 	if _, err := ValidateKeyID(o.KeyID); err != nil {
 		return fmt.Errorf("failed to validate key id: %w", err)
@@ -314,15 +357,15 @@ func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
 	return nil
 }
 
-// validateEncryptedDEK tests the following:
-// 1. The encrypted DEK is not empty.
-// 2. The size of encrypted DEK is less than 1 kB.
-func validateEncryptedDEK(encryptedDEK []byte) error {
-	if len(encryptedDEK) == 0 {
-		return fmt.Errorf("encrypted DEK is empty")
+// validateEncryptedDEKSource tests the following:
+// 1. The encrypted DEK source is not empty.
+// 2. The size of encrypted DEK source is less than 1 kB.
+func validateEncryptedDEKSource(encryptedDEKSource []byte) error {
+	if len(encryptedDEKSource) == 0 {
+		return fmt.Errorf("encrypted DEK source is empty")
 	}
-	if len(encryptedDEK) > encryptedDEKMaxSize {
-		return fmt.Errorf("encrypted DEK is %d bytes, which exceeds the max size of %d", len(encryptedDEK), encryptedDEKMaxSize)
+	if len(encryptedDEKSource) > encryptedDEKSourceMaxSize {
+		return fmt.Errorf("encrypted DEK source is %d bytes, which exceeds the max size of %d", len(encryptedDEKSource), encryptedDEKSourceMaxSize)
 	}
 	return nil
 }
@@ -367,17 +410,19 @@ func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestIn
 
 // generateCacheKey returns a key for the cache.
 // The key is a concatenation of:
-//  1. encryptedDEK
+//  0. encryptedDEKSourceType
+//  1. encryptedDEKSource
 //  2. keyID
 //  3. length of annotations
 //  4. annotations (sorted by key) - each annotation is a concatenation of:
 //     a. annotation key
 //     b. annotation value
-func generateCacheKey(encryptedDEK []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
+func generateCacheKey(encryptedDEKSourceType kmstypes.EncryptedDEKSourceType, encryptedDEKSource []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
 	// TODO(aramase): use sync pool buffer to avoid allocations
 	b := cryptobyte.NewBuilder(nil)
+	b.AddUint32(uint32(encryptedDEKSourceType))
 	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
-		b.AddBytes(encryptedDEK)
+		b.AddBytes(encryptedDEKSource)
 	})
 	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
 		b.AddBytes(toBytes(keyID))
@@ -420,3 +465,11 @@ func toBytes(s string) []byte {
 	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/os/file.go#L246
 	return unsafe.Slice(unsafe.StringData(s), len(s))
 }
+
+// GetHashIfNotEmpty returns the sha256 hash of the data if it is not empty.
+func GetHashIfNotEmpty(data string) string {
+	if len(data) > 0 {
+		return fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(data)))
+	}
+	return ""
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.pb.go

@@ -36,19 +36,52 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
 
+type EncryptedDEKSourceType int32
+
+const (
+	// AES_GCM_KEY means that the plaintext of encryptedDEKSource is the DEK itself, with AES-GCM as the encryption algorithm.
+	EncryptedDEKSourceType_AES_GCM_KEY EncryptedDEKSourceType = 0
+	// HKDF_SHA256_XNONCE_AES_GCM_SEED means that the plaintext of encryptedDEKSource is the pseudo random key
+	// (referred to as the seed throughout the code) that is fed into HKDF expand.  SHA256 is the hash algorithm
+	// and first 32 bytes of encryptedData are the info param.  The first 32 bytes from the HKDF stream are used
+	// as the DEK with AES-GCM as the encryption algorithm.
+	EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED EncryptedDEKSourceType = 1
+)
+
+var EncryptedDEKSourceType_name = map[int32]string{
+	0: "AES_GCM_KEY",
+	1: "HKDF_SHA256_XNONCE_AES_GCM_SEED",
+}
+
+var EncryptedDEKSourceType_value = map[string]int32{
+	"AES_GCM_KEY":                     0,
+	"HKDF_SHA256_XNONCE_AES_GCM_SEED": 1,
+}
+
+func (x EncryptedDEKSourceType) String() string {
+	return proto.EnumName(EncryptedDEKSourceType_name, int32(x))
+}
+
+func (EncryptedDEKSourceType) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_00212fb1f9d3bf1c, []int{0}
+}
+
 // EncryptedObject is the representation of data stored in etcd after envelope encryption.
 type EncryptedObject struct {
 	// EncryptedData is the encrypted data.
 	EncryptedData []byte `protobuf:"bytes,1,opt,name=encryptedData,proto3" json:"encryptedData,omitempty"`
 	// KeyID is the KMS key ID used for encryption operations.
 	KeyID string `protobuf:"bytes,2,opt,name=keyID,proto3" json:"keyID,omitempty"`
-	// EncryptedDEK is the encrypted DEK.
-	EncryptedDEK []byte `protobuf:"bytes,3,opt,name=encryptedDEK,proto3" json:"encryptedDEK,omitempty"`
+	// EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
+	// encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
+	EncryptedDEKSource []byte `protobuf:"bytes,3,opt,name=encryptedDEKSource,proto3" json:"encryptedDEKSource,omitempty"`
 	// Annotations is additional metadata that was provided by the KMS plugin.
-	Annotations          map[string][]byte `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
-	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
-	XXX_unrecognized     []byte            `json:"-"`
-	XXX_sizecache        int32             `json:"-"`
+	Annotations map[string][]byte `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.
+	EncryptedDEKSourceType EncryptedDEKSourceType `protobuf:"varint,5,opt,name=encryptedDEKSourceType,proto3,enum=v2.EncryptedDEKSourceType" json:"encryptedDEKSourceType,omitempty"`
+	XXX_NoUnkeyedLiteral   struct{}               `json:"-"`
+	XXX_unrecognized       []byte                 `json:"-"`
+	XXX_sizecache          int32                  `json:"-"`
 }
 
 func (m *EncryptedObject) Reset()         { *m = EncryptedObject{} }
@@ -89,9 +122,9 @@ func (m *EncryptedObject) GetKeyID() string {
 	return ""
 }
 
-func (m *EncryptedObject) GetEncryptedDEK() []byte {
+func (m *EncryptedObject) GetEncryptedDEKSource() []byte {
 	if m != nil {
-		return m.EncryptedDEK
+		return m.EncryptedDEKSource
 	}
 	return nil
 }
@@ -103,7 +136,15 @@ func (m *EncryptedObject) GetAnnotations() map[string][]byte {
 	return nil
 }
 
+func (m *EncryptedObject) GetEncryptedDEKSourceType() EncryptedDEKSourceType {
+	if m != nil {
+		return m.EncryptedDEKSourceType
+	}
+	return EncryptedDEKSourceType_AES_GCM_KEY
+}
+
 func init() {
+	proto.RegisterEnum("v2.EncryptedDEKSourceType", EncryptedDEKSourceType_name, EncryptedDEKSourceType_value)
 	proto.RegisterType((*EncryptedObject)(nil), "v2.EncryptedObject")
 	proto.RegisterMapType((map[string][]byte)(nil), "v2.EncryptedObject.AnnotationsEntry")
 }
@@ -111,21 +152,26 @@ func init() {
 func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
 
 var fileDescriptor_00212fb1f9d3bf1c = []byte{
-	// 244 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x90, 0xb1, 0x4b, 0x03, 0x31,
-	0x14, 0xc6, 0xc9, 0x9d, 0x0a, 0x97, 0x9e, 0x58, 0x82, 0xc3, 0xe1, 0x74, 0x94, 0x0e, 0x37, 0x25,
-	0x10, 0x97, 0x22, 0x52, 0x50, 0x7a, 0x82, 0x38, 0x08, 0x19, 0xdd, 0xd2, 0xfa, 0x28, 0x67, 0x6a,
-	0x12, 0x92, 0x18, 0xc8, 0x9f, 0xee, 0x26, 0x4d, 0x95, 0xda, 0xdb, 0xde, 0xf7, 0xf1, 0xfb, 0xe0,
-	0xc7, 0xc3, 0x95, 0xb4, 0x03, 0xb5, 0xce, 0x04, 0x43, 0x8a, 0xc8, 0x67, 0xdf, 0x08, 0x5f, 0xf5,
-	0x7a, 0xe3, 0x92, 0x0d, 0xf0, 0xfe, 0xba, 0xfe, 0x80, 0x4d, 0x20, 0x73, 0x7c, 0x09, 0x7f, 0xd5,
-	0x4a, 0x06, 0xd9, 0xa0, 0x16, 0x75, 0xb5, 0x38, 0x2d, 0xc9, 0x35, 0x3e, 0x57, 0x90, 0x9e, 0x57,
-	0x4d, 0xd1, 0xa2, 0xae, 0x12, 0x87, 0x40, 0x66, 0xb8, 0x3e, 0x62, 0xfd, 0x4b, 0x53, 0xe6, 0xe9,
-	0x49, 0x47, 0x9e, 0xf0, 0x44, 0x6a, 0x6d, 0x82, 0x0c, 0x83, 0xd1, 0xbe, 0x39, 0x6b, 0xcb, 0x6e,
-	0xc2, 0xe7, 0x34, 0x72, 0x3a, 0x32, 0xa1, 0x0f, 0x47, 0xac, 0xd7, 0xc1, 0x25, 0xf1, 0x7f, 0x78,
-	0xb3, 0xc4, 0xd3, 0x31, 0x40, 0xa6, 0xb8, 0x54, 0x90, 0xb2, 0x71, 0x25, 0xf6, 0xe7, 0xde, 0x33,
-	0xca, 0xdd, 0x17, 0x64, 0xcf, 0x5a, 0x1c, 0xc2, 0x5d, 0xb1, 0x40, 0x8f, 0xcb, 0xb7, 0x7b, 0xb5,
-	0xf0, 0x74, 0x30, 0x4c, 0xda, 0xc1, 0x83, 0x8b, 0xe0, 0x98, 0x55, 0x5b, 0xe6, 0x83, 0x71, 0x72,
-	0x0b, 0x2c, 0x93, 0xec, 0x57, 0x9d, 0x81, 0x8e, 0xb0, 0x33, 0x16, 0x98, 0xfa, 0xf4, 0x91, 0xb3,
-	0xc8, 0xd7, 0x17, 0xf9, 0x8d, 0xb7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x80, 0x43, 0x93,
-	0x53, 0x01, 0x00, 0x00,
+	// 329 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0xe1, 0x4b, 0xc2, 0x40,
+	0x18, 0xc6, 0xdb, 0xcc, 0xc0, 0xd3, 0x72, 0x1c, 0x21, 0xc3, 0x2f, 0x8d, 0xf2, 0xc3, 0xe8, 0xc3,
+	0x0e, 0x16, 0x85, 0x44, 0x08, 0xe6, 0xce, 0x0c, 0x49, 0x61, 0xeb, 0x43, 0xf5, 0x65, 0x9c, 0xf6,
+	0x22, 0x6b, 0xb6, 0x1b, 0xb7, 0xf3, 0x60, 0x7f, 0x6a, 0xff, 0x4d, 0x38, 0x13, 0xd3, 0xec, 0xdb,
+	0xbd, 0xef, 0xfd, 0xde, 0xe7, 0xb9, 0x7b, 0x5e, 0x54, 0x61, 0x69, 0xe4, 0xa4, 0x82, 0x4b, 0x8e,
+	0x75, 0xe5, 0x9e, 0x7f, 0xe9, 0xa8, 0x4e, 0x93, 0xa9, 0xc8, 0x53, 0x09, 0xef, 0xe3, 0xc9, 0x07,
+	0x4c, 0x25, 0x6e, 0xa1, 0x63, 0x58, 0xb7, 0x3c, 0x26, 0x99, 0xa9, 0x59, 0x9a, 0x5d, 0xf3, 0xb7,
+	0x9b, 0xf8, 0x14, 0x95, 0x63, 0xc8, 0x1f, 0x3d, 0x53, 0xb7, 0x34, 0xbb, 0xe2, 0xaf, 0x0a, 0xec,
+	0x20, 0xbc, 0xc1, 0xe8, 0x30, 0xe0, 0x0b, 0x31, 0x05, 0xb3, 0x54, 0x08, 0xec, 0xb9, 0xc1, 0x7d,
+	0x54, 0x65, 0x49, 0xc2, 0x25, 0x93, 0x11, 0x4f, 0x32, 0xf3, 0xd0, 0x2a, 0xd9, 0x55, 0xb7, 0xe5,
+	0x28, 0xd7, 0xd9, 0x79, 0x95, 0xd3, 0xdd, 0x60, 0x34, 0x91, 0x22, 0xf7, 0x7f, 0x0f, 0x62, 0x1f,
+	0x35, 0xfe, 0xaa, 0x3f, 0xe7, 0x29, 0x98, 0x65, 0x4b, 0xb3, 0x4f, 0xdc, 0xe6, 0x96, 0xe4, 0x16,
+	0xe1, 0xff, 0x33, 0xd9, 0xec, 0x20, 0x63, 0xd7, 0x14, 0x1b, 0xa8, 0x14, 0x43, 0x5e, 0x24, 0x52,
+	0xf1, 0x97, 0xc7, 0x65, 0x0e, 0x8a, 0xcd, 0x17, 0x50, 0xe4, 0x50, 0xf3, 0x57, 0xc5, 0xad, 0xde,
+	0xd6, 0x2e, 0x47, 0xa8, 0xb1, 0xdf, 0x11, 0xd7, 0x51, 0xb5, 0x4b, 0x83, 0xf0, 0xa1, 0xf7, 0x14,
+	0x0e, 0xe9, 0xab, 0x71, 0x80, 0x2f, 0xd0, 0xd9, 0x60, 0xe8, 0xf5, 0xc3, 0x60, 0xd0, 0x75, 0xaf,
+	0x6f, 0xc2, 0x97, 0xd1, 0x78, 0xd4, 0xa3, 0xe1, 0x9a, 0x09, 0x28, 0xf5, 0x0c, 0xed, 0xbe, 0xf3,
+	0x76, 0x17, 0xb7, 0x33, 0x27, 0xe2, 0x84, 0xa5, 0x51, 0x06, 0x42, 0x81, 0x20, 0x69, 0x3c, 0x23,
+	0x99, 0xe4, 0x82, 0xcd, 0x80, 0x14, 0xce, 0xe4, 0xe7, 0x33, 0x04, 0x12, 0x05, 0x73, 0x9e, 0x02,
+	0x89, 0x3f, 0x33, 0xe5, 0x12, 0xe5, 0x4e, 0x8e, 0x8a, 0xb5, 0x5f, 0x7d, 0x07, 0x00, 0x00, 0xff,
+	0xff, 0xcc, 0x0f, 0x2b, 0x2e, 0x03, 0x02, 0x00, 0x00,
 }

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.proto

@@ -28,9 +28,24 @@ message EncryptedObject {
   // KeyID is the KMS key ID used for encryption operations.
   string keyID = 2;
 
-  // EncryptedDEK is the encrypted DEK.
-  bytes encryptedDEK = 3;
+  // EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
+  // encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
+  bytes encryptedDEKSource = 3;
 
   // Annotations is additional metadata that was provided by the KMS plugin.
   map<string, bytes> annotations = 4;
+
+  // encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.
+  EncryptedDEKSourceType encryptedDEKSourceType = 5;
+}
+
+enum EncryptedDEKSourceType {
+  // AES_GCM_KEY means that the plaintext of encryptedDEKSource is the DEK itself, with AES-GCM as the encryption algorithm.
+  AES_GCM_KEY = 0;
+
+  // HKDF_SHA256_XNONCE_AES_GCM_SEED means that the plaintext of encryptedDEKSource is the pseudo random key
+  // (referred to as the seed throughout the code) that is fed into HKDF expand.  SHA256 is the hash algorithm
+  // and first 32 bytes of encryptedData are the info param.  The first 32 bytes from the HKDF stream are used
+  // as the DEK with AES-GCM as the encryption algorithm.
+  HKDF_SHA256_XNONCE_AES_GCM_SEED = 1;
 }

vendor/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -17,9 +17,11 @@ limitations under the License.
 package value
 
 import (
+	"errors"
 	"sync"
 	"time"
 
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
 	"k8s.io/component-base/metrics"
@@ -59,7 +61,7 @@ var (
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "transformation_operations_total",
-			Help:           "Total number of transformations.",
+			Help:           "Total number of transformations. Successful transformation will have a status 'OK' and a varied status string when the transformation fails. This status and transformation_type fields may be used for alerting on encryption/decryption failure using transformation_type from_storage for decryption and to_storage for encryption",
 			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"transformation_type", "transformer_prefix", "status"},
@@ -112,7 +114,7 @@ func RegisterMetrics() {
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
 // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
 func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) {
-	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, status.Code(err).String()).Inc()
+	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, getErrorCode(err)).Inc()
 
 	if err == nil {
 		transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds())
@@ -138,3 +140,23 @@ func RecordDataKeyGeneration(start time.Time, err error) {
 func sinceInSeconds(start time.Time) float64 {
 	return time.Since(start).Seconds()
 }
+
+type gRPCError interface {
+	GRPCStatus() *status.Status
+}
+
+func getErrorCode(err error) string {
+	if err == nil {
+		return codes.OK.String()
+	}
+
+	// handle errors wrapped with fmt.Errorf and similar
+	var s gRPCError
+	if errors.As(err, &s) {
+		return s.GRPCStatus().Code().String()
+	}
+
+	// This is not gRPC error. The operation must have failed before gRPC
+	// method was called, otherwise we would get gRPC error.
+	return "unknown-non-grpc"
+}

vendor/k8s.io/apiserver/pkg/storage/value/transformer.go

@@ -23,7 +23,10 @@ import (
 	"fmt"
 	"time"
 
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/errors"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog/v2"
 )
 
 func init() {
@@ -39,17 +42,30 @@ type Context interface {
 	AuthenticatedData() []byte
 }
 
-// Transformer allows a value to be transformed before being read from or written to the underlying store. The methods
-// must be able to undo the transformation caused by the other.
-type Transformer interface {
+type Read interface {
 	// TransformFromStorage may transform the provided data from its underlying storage representation or return an error.
 	// Stale is true if the object on disk is stale and a write to etcd should be issued, even if the contents of the object
 	// have not changed.
 	TransformFromStorage(ctx context.Context, data []byte, dataCtx Context) (out []byte, stale bool, err error)
+}
+
+type Write interface {
 	// TransformToStorage may transform the provided data into the appropriate form in storage or return an error.
 	TransformToStorage(ctx context.Context, data []byte, dataCtx Context) (out []byte, err error)
 }
 
+// Transformer allows a value to be transformed before being read from or written to the underlying store. The methods
+// must be able to undo the transformation caused by the other.
+type Transformer interface {
+	Read
+	Write
+}
+
+// ResourceTransformers returns a transformer for the provided resource.
+type ResourceTransformers interface {
+	TransformerForResource(resource schema.GroupResource) Transformer
+}
+
 // DefaultContext is a simple implementation of Context for a slice of bytes.
 type DefaultContext []byte
 
@@ -144,6 +160,7 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
 		}
 	}
 	if err := errors.Reduce(errors.NewAggregate(errs)); err != nil {
+		logTransformErr(ctx, err, "failed to decrypt data")
 		return nil, false, err
 	}
 	RecordTransformation("from_storage", "unknown", time.Since(start), t.err)
@@ -157,6 +174,7 @@ func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte
 	result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx)
 	RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err)
 	if err != nil {
+		logTransformErr(ctx, err, "failed to encrypt data")
 		return nil, err
 	}
 	prefixedData := make([]byte, len(transformer.Prefix), len(result)+len(transformer.Prefix))
@@ -164,3 +182,32 @@ func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte
 	prefixedData = append(prefixedData, result...)
 	return prefixedData, nil
 }
+
+func logTransformErr(ctx context.Context, err error, message string) {
+	requestInfo := getRequestInfoFromContext(ctx)
+	if klogLevel6 := klog.V(6); klogLevel6.Enabled() {
+		klogLevel6.InfoSDepth(
+			1,
+			message,
+			"err", err,
+			"group", requestInfo.APIGroup,
+			"version", requestInfo.APIVersion,
+			"resource", requestInfo.Resource,
+			"subresource", requestInfo.Subresource,
+			"verb", requestInfo.Verb,
+			"namespace", requestInfo.Namespace,
+			"name", requestInfo.Name,
+		)
+
+		return
+	}
+
+	klog.ErrorSDepth(1, err, message)
+}
+
+func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestInfo {
+	if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
+		return reqInfo
+	}
+	return &genericapirequest.RequestInfo{}
+}

vendor/k8s.io/apiserver/pkg/storageversion/manager.go

@@ -44,6 +44,10 @@ type ResourceInfo struct {
 	// DirectlyDecodableVersions is a list of versions that the converter for REST storage knows how to convert.  This
 	// contains items like apiextensions.k8s.io/v1beta1 even if we don't serve that version.
 	DirectlyDecodableVersions []schema.GroupVersion
+
+	// ServedVersions holds a list of all versions of GroupResource that are served.  Note that a server may be able to
+	// decode a particular version, but still not serve it.
+	ServedVersions []string
 }
 
 // Manager records the resources whose StorageVersions need updates, and provides a method to update those StorageVersions.
@@ -143,7 +147,10 @@ func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.C
 		if len(gr.Group) == 0 {
 			gr.Group = "core"
 		}
-		if err := updateStorageVersionFor(sc, serverID, gr, r.EncodingVersion, decodableVersions); err != nil {
+
+		servedVersions := r.ServedVersions
+
+		if err := updateStorageVersionFor(sc, serverID, gr, r.EncodingVersion, decodableVersions, servedVersions); err != nil {
 			utilruntime.HandleError(fmt.Errorf("failed to update storage version for %v: %v", r.GroupResource, err))
 			s.recordStatusFailure(&r, err)
 			hasFailure = true