rebase: update k8s.io packages to v0.29.0

Signed-off-by: Niels de Vos <ndevos@ibm.com>

vendor/k8s.io/apiserver/pkg/admission/config.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -60,7 +59,7 @@ func ReadAdmissionConfiguration(pluginNames []string, configFilePath string, con
 		return configProvider{config: &apiserver.AdmissionConfiguration{}}, nil
 	}
 	// a file was provided, so we just read it.
-	data, err := ioutil.ReadFile(configFilePath)
+	data, err := os.ReadFile(configFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read admission control configuration from %q [%v]", configFilePath, err)
 	}
@@ -141,7 +140,7 @@ func GetAdmissionPluginConfigurationFor(pluginCfg apiserver.AdmissionPluginConfi
 	}
 	// there is nothing nested, so we delegate to path
 	if pluginCfg.Path != "" {
-		content, err := ioutil.ReadFile(pluginCfg.Path)
+		content, err := os.ReadFile(pluginCfg.Path)
 		if err != nil {
 			klog.Fatalf("Couldn't open admission plugin configuration %s: %#v", pluginCfg.Path, err)
 			return nil, err

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/compile.go

@@ -141,6 +141,7 @@ type CompilationResult struct {
 	Program            cel.Program
 	Error              *apiservercel.Error
 	ExpressionAccessor ExpressionAccessor
+	OutputType         *cel.Type
 }
 
 // Compiler provides a CEL expression compiler configured with the desired admission related CEL variables and
@@ -214,6 +215,7 @@ func (c compiler) CompileCELExpression(expressionAccessor ExpressionAccessor, op
 	return CompilationResult{
 		Program:            prog,
 		ExpressionAccessor: expressionAccessor,
+		OutputType:         ast.OutputType(),
 	}
 }
 

vendor/k8s.io/apiserver/pkg/admission/plugin/cel/composition.go

@@ -23,6 +23,7 @@ import (
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
+	"github.com/google/cel-go/common/types/traits"
 
 	v1 "k8s.io/api/admission/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -69,8 +70,8 @@ func (c *CompositedCompiler) CompileAndStoreVariables(variables []NamedExpressio
 }
 
 func (c *CompositedCompiler) CompileAndStoreVariable(variable NamedExpressionAccessor, options OptionalVariableDeclarations, mode environment.Type) CompilationResult {
-	c.CompositionEnv.AddField(variable.GetName())
 	result := c.Compiler.CompileCELExpression(variable, options, mode)
+	c.CompositionEnv.AddField(variable.GetName(), result.OutputType)
 	c.CompositionEnv.CompiledVariables[variable.GetName()] = result
 	return result
 }
@@ -90,8 +91,8 @@ type CompositionEnv struct {
 	CompiledVariables map[string]CompilationResult
 }
 
-func (c *CompositionEnv) AddField(name string) {
-	c.MapType.Fields[name] = apiservercel.NewDeclField(name, apiservercel.DynType, true, nil, nil)
+func (c *CompositionEnv) AddField(name string, celType *cel.Type) {
+	c.MapType.Fields[name] = apiservercel.NewDeclField(name, convertCelTypeToDeclType(celType), true, nil, nil)
 }
 
 func NewCompositionEnv(typeName string, baseEnvSet *environment.EnvSet) (*CompositionEnv, error) {
@@ -196,3 +197,48 @@ func (a *variableAccessor) Callback(_ *lazy.MapValue) ref.Val {
 	}
 	return v
 }
+
+// convertCelTypeToDeclType converts a cel.Type to DeclType, for the use of
+// the TypeProvider and the cost estimator.
+// List and map types are created on-demand with their parameters converted recursively.
+func convertCelTypeToDeclType(celType *cel.Type) *apiservercel.DeclType {
+	if celType == nil {
+		return apiservercel.DynType
+	}
+	switch celType {
+	case cel.AnyType:
+		return apiservercel.AnyType
+	case cel.BoolType:
+		return apiservercel.BoolType
+	case cel.BytesType:
+		return apiservercel.BytesType
+	case cel.DoubleType:
+		return apiservercel.DoubleType
+	case cel.DurationType:
+		return apiservercel.DurationType
+	case cel.IntType:
+		return apiservercel.IntType
+	case cel.NullType:
+		return apiservercel.NullType
+	case cel.StringType:
+		return apiservercel.StringType
+	case cel.TimestampType:
+		return apiservercel.TimestampType
+	case cel.UintType:
+		return apiservercel.UintType
+	default:
+		if celType.HasTrait(traits.ContainerType) && celType.HasTrait(traits.IndexerType) {
+			parameters := celType.Parameters()
+			switch len(parameters) {
+			case 1:
+				elemType := convertCelTypeToDeclType(parameters[0])
+				return apiservercel.NewListType(elemType, -1)
+			case 2:
+				keyType := convertCelTypeToDeclType(parameters[0])
+				valueType := convertCelTypeToDeclType(parameters[1])
+				return apiservercel.NewMapType(keyType, valueType, -1)
+			}
+		}
+		return apiservercel.DynType
+	}
+}

vendor/k8s.io/apiserver/pkg/admission/plugin/validatingadmissionpolicy/typechecking.go

@@ -238,7 +238,7 @@ func (c *TypeChecker) typesToCheck(p *v1beta1.ValidatingAdmissionPolicy) []schem
 	if p.Spec.MatchConstraints == nil || len(p.Spec.MatchConstraints.ResourceRules) == 0 {
 		return nil
 	}
-
+	restMapperRefreshAttempted := false // at most once per policy, refresh RESTMapper and retry resolution.
 	for _, rule := range p.Spec.MatchConstraints.ResourceRules {
 		groups := extractGroups(&rule.Rule)
 		if len(groups) == 0 {
@@ -268,7 +268,16 @@ func (c *TypeChecker) typesToCheck(p *v1beta1.ValidatingAdmissionPolicy) []schem
 					}
 					resolved, err := c.RestMapper.KindsFor(gvr)
 					if err != nil {
-						continue
+						if restMapperRefreshAttempted {
+							// RESTMapper refresh happens at most once per policy
+							continue
+						}
+						c.tryRefreshRESTMapper()
+						restMapperRefreshAttempted = true
+						resolved, err = c.RestMapper.KindsFor(gvr)
+						if err != nil {
+							continue
+						}
 					}
 					for _, r := range resolved {
 						if !r.Empty() {
@@ -344,6 +353,13 @@ func sortGVKList(list []schema.GroupVersionKind) []schema.GroupVersionKind {
 	return list
 }
 
+// tryRefreshRESTMapper refreshes the RESTMapper if it supports refreshing.
+func (c *TypeChecker) tryRefreshRESTMapper() {
+	if r, ok := c.RestMapper.(meta.ResettableRESTMapper); ok {
+		r.Reset()
+	}
+}
+
 func buildEnv(hasParams bool, hasAuthorizer bool, types typeOverwrite) (*cel.Env, error) {
 	baseEnv := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion())
 	requestType := plugincel.BuildRequestType()

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/config/kubeconfig.go

@@ -19,7 +19,6 @@ package config
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -47,7 +46,7 @@ func LoadConfig(configFile io.Reader) (string, error) {
 	var kubeconfigFile string
 	if configFile != nil {
 		// we have a config so parse it.
-		data, err := ioutil.ReadAll(configFile)
+		data, err := io.ReadAll(configFile)
 		if err != nil {
 			return "", err
 		}

vendor/k8s.io/apiserver/pkg/admission/plugins.go

@@ -20,7 +20,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"reflect"
 	"sort"
 	"strings"
@@ -115,7 +114,7 @@ func splitStream(config io.Reader) (io.Reader, io.Reader, error) {
 		return nil, nil, nil
 	}
 
-	configBytes, err := ioutil.ReadAll(config)
+	configBytes, err := io.ReadAll(config)
 	if err != nil {
 		return nil, nil, err
 	}

vendor/k8s.io/apiserver/pkg/apis/apiserver/register.go

@@ -43,6 +43,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 	)
 	scheme.AddKnownTypes(SchemeGroupVersion,
 		&AdmissionConfiguration{},
+		&AuthenticationConfiguration{},
+		&AuthorizationConfiguration{},
 		&EgressSelectorConfiguration{},
 		&TracingConfiguration{},
 	)

vendor/k8s.io/apiserver/pkg/apis/apiserver/types.go

@@ -157,3 +157,188 @@ type TracingConfiguration struct {
 	// Embed the component config tracing configuration struct
 	tracingapi.TracingConfiguration
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// AuthenticationConfiguration provides versioned configuration for authentication.
+type AuthenticationConfiguration struct {
+	metav1.TypeMeta
+
+	JWT []JWTAuthenticator
+}
+
+// JWTAuthenticator provides the configuration for a single JWT authenticator.
+type JWTAuthenticator struct {
+	Issuer               Issuer
+	ClaimValidationRules []ClaimValidationRule
+	ClaimMappings        ClaimMappings
+	UserValidationRules  []UserValidationRule
+}
+
+// Issuer provides the configuration for a external provider specific settings.
+type Issuer struct {
+	URL                  string
+	CertificateAuthority string
+	Audiences            []string
+}
+
+// ClaimValidationRule provides the configuration for a single claim validation rule.
+type ClaimValidationRule struct {
+	Claim         string
+	RequiredValue string
+
+	Expression string
+	Message    string
+}
+
+// ClaimMappings provides the configuration for claim mapping
+type ClaimMappings struct {
+	Username PrefixedClaimOrExpression
+	Groups   PrefixedClaimOrExpression
+	UID      ClaimOrExpression
+	Extra    []ExtraMapping
+}
+
+// PrefixedClaimOrExpression provides the configuration for a single prefixed claim or expression.
+type PrefixedClaimOrExpression struct {
+	Claim  string
+	Prefix *string
+
+	Expression string
+}
+
+// ClaimOrExpression provides the configuration for a single claim or expression.
+type ClaimOrExpression struct {
+	Claim      string
+	Expression string
+}
+
+// ExtraMapping provides the configuration for a single extra mapping.
+type ExtraMapping struct {
+	Key             string
+	ValueExpression string
+}
+
+// UserValidationRule provides the configuration for a single user validation rule.
+type UserValidationRule struct {
+	Expression string
+	Message    string
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type AuthorizationConfiguration struct {
+	metav1.TypeMeta
+
+	// Authorizers is an ordered list of authorizers to
+	// authorize requests against.
+	// This is similar to the --authorization-modes kube-apiserver flag
+	// Must be at least one.
+	Authorizers []AuthorizerConfiguration `json:"authorizers"`
+}
+
+const (
+	TypeWebhook                                          AuthorizerType = "Webhook"
+	FailurePolicyNoOpinion                               string         = "NoOpinion"
+	FailurePolicyDeny                                    string         = "Deny"
+	AuthorizationWebhookConnectionInfoTypeKubeConfigFile string         = "KubeConfigFile"
+	AuthorizationWebhookConnectionInfoTypeInCluster      string         = "InClusterConfig"
+)
+
+type AuthorizerType string
+
+type AuthorizerConfiguration struct {
+	// Type refers to the type of the authorizer
+	// "Webhook" is supported in the generic API server
+	// Other API servers may support additional authorizer
+	// types like Node, RBAC, ABAC, etc.
+	Type AuthorizerType
+
+	// Name used to describe the webhook
+	// This is explicitly used in monitoring machinery for metrics
+	// Note: Names must be DNS1123 labels like `myauthorizername` or
+	//		 subdomains like `myauthorizer.example.domain`
+	// Required, with no default
+	Name string
+
+	// Webhook defines the configuration for a Webhook authorizer
+	// Must be defined when Type=Webhook
+	Webhook *WebhookConfiguration
+}
+
+type WebhookConfiguration struct {
+	// The duration to cache 'authorized' responses from the webhook
+	// authorizer.
+	// Same as setting `--authorization-webhook-cache-authorized-ttl` flag
+	// Default: 5m0s
+	AuthorizedTTL metav1.Duration
+	// The duration to cache 'unauthorized' responses from the webhook
+	// authorizer.
+	// Same as setting `--authorization-webhook-cache-unauthorized-ttl` flag
+	// Default: 30s
+	UnauthorizedTTL metav1.Duration
+	// Timeout for the webhook request
+	// Maximum allowed value is 30s.
+	// Required, no default value.
+	Timeout metav1.Duration
+	// The API version of the authorization.k8s.io SubjectAccessReview to
+	// send to and expect from the webhook.
+	// Same as setting `--authorization-webhook-version` flag
+	// Valid values: v1beta1, v1
+	// Required, no default value
+	SubjectAccessReviewVersion string
+	// MatchConditionSubjectAccessReviewVersion specifies the SubjectAccessReview
+	// version the CEL expressions are evaluated against
+	// Valid values: v1
+	// Required, no default value
+	MatchConditionSubjectAccessReviewVersion string
+	// Controls the authorization decision when a webhook request fails to
+	// complete or returns a malformed response or errors evaluating
+	// matchConditions.
+	// Valid values:
+	//   - NoOpinion: continue to subsequent authorizers to see if one of
+	//     them allows the request
+	//   - Deny: reject the request without consulting subsequent authorizers
+	// Required, with no default.
+	FailurePolicy string
+
+	// ConnectionInfo defines how we talk to the webhook
+	ConnectionInfo WebhookConnectionInfo
+
+	// matchConditions is a list of conditions that must be met for a request to be sent to this
+	// webhook. An empty list of matchConditions matches all requests.
+	// There are a maximum of 64 match conditions allowed.
+	//
+	// The exact matching logic is (in order):
+	//   1. If at least one matchCondition evaluates to FALSE, then the webhook is skipped.
+	//   2. If ALL matchConditions evaluate to TRUE, then the webhook is called.
+	//   3. If at least one matchCondition evaluates to an error (but none are FALSE):
+	//      - If failurePolicy=Deny, then the webhook rejects the request
+	//      - If failurePolicy=NoOpinion, then the error is ignored and the webhook is skipped
+	MatchConditions []WebhookMatchCondition
+}
+
+type WebhookConnectionInfo struct {
+	// Controls how the webhook should communicate with the server.
+	// Valid values:
+	// - KubeConfigFile: use the file specified in kubeConfigFile to locate the
+	//   server.
+	// - InClusterConfig: use the in-cluster configuration to call the
+	//   SubjectAccessReview API hosted by kube-apiserver. This mode is not
+	//   allowed for kube-apiserver.
+	Type string
+
+	// Path to KubeConfigFile for connection info
+	// Required, if connectionInfo.Type is KubeConfig
+	KubeConfigFile *string
+}
+
+type WebhookMatchCondition struct {
+	// expression represents the expression which will be evaluated by CEL. Must evaluate to bool.
+	// CEL expressions have access to the contents of the SubjectAccessReview in v1 version.
+	// If version specified by subjectAccessReviewVersion in the request variable is v1beta1,
+	// the contents would be converted to the v1 version before evaluating the CEL expression.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	Expression string
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/defaults.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+func addDefaultingFuncs(scheme *runtime.Scheme) error {
+	return RegisterDefaults(scheme)
+}
+
+func SetDefaults_WebhookConfiguration(obj *WebhookConfiguration) {
+	if obj.AuthorizedTTL.Duration == 0 {
+		obj.AuthorizedTTL.Duration = 5 * time.Minute
+	}
+	if obj.UnauthorizedTTL.Duration == 0 {
+		obj.UnauthorizedTTL.Duration = 30 * time.Second
+	}
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/register.go

@@ -43,7 +43,7 @@ func init() {
 	// We only register manually written functions here. The registration of the
 	// generated functions takes place in the generated files. The separation
 	// makes the code compile even when the generated files are missing.
-	localSchemeBuilder.Register(addKnownTypes)
+	localSchemeBuilder.Register(addKnownTypes, addDefaultingFuncs)
 }
 
 // Adds the list of known types to the given scheme.
@@ -53,6 +53,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&EgressSelectorConfiguration{},
 	)
 	scheme.AddKnownTypes(ConfigSchemeGroupVersion,
+		&AuthenticationConfiguration{},
+		&AuthorizationConfiguration{},
 		&TracingConfiguration{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go

@@ -158,3 +158,379 @@ type TracingConfiguration struct {
 	// Embed the component config tracing configuration struct
 	tracingapi.TracingConfiguration `json:",inline"`
 }
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// AuthenticationConfiguration provides versioned configuration for authentication.
+type AuthenticationConfiguration struct {
+	metav1.TypeMeta
+
+	// jwt is a list of authenticator to authenticate Kubernetes users using
+	// JWT compliant tokens. The authenticator will attempt to parse a raw ID token,
+	// verify it's been signed by the configured issuer. The public key to verify the
+	// signature is discovered from the issuer's public endpoint using OIDC discovery.
+	// For an incoming token, each JWT authenticator will be attempted in
+	// the order in which it is specified in this list.  Note however that
+	// other authenticators may run before or after the JWT authenticators.
+	// The specific position of JWT authenticators in relation to other
+	// authenticators is neither defined nor stable across releases.  Since
+	// each JWT authenticator must have a unique issuer URL, at most one
+	// JWT authenticator will attempt to cryptographically validate the token.
+	JWT []JWTAuthenticator `json:"jwt"`
+}
+
+// JWTAuthenticator provides the configuration for a single JWT authenticator.
+type JWTAuthenticator struct {
+	// issuer contains the basic OIDC provider connection options.
+	// +required
+	Issuer Issuer `json:"issuer"`
+
+	// claimValidationRules are rules that are applied to validate token claims to authenticate users.
+	// +optional
+	ClaimValidationRules []ClaimValidationRule `json:"claimValidationRules,omitempty"`
+
+	// claimMappings points claims of a token to be treated as user attributes.
+	// +required
+	ClaimMappings ClaimMappings `json:"claimMappings"`
+
+	// userValidationRules are rules that are applied to final user before completing authentication.
+	// These allow invariants to be applied to incoming identities such as preventing the
+	// use of the system: prefix that is commonly used by Kubernetes components.
+	// The validation rules are logically ANDed together and must all return true for the validation to pass.
+	// +optional
+	UserValidationRules []UserValidationRule `json:"userValidationRules,omitempty"`
+}
+
+// Issuer provides the configuration for a external provider specific settings.
+type Issuer struct {
+	// url points to the issuer URL in a format https://url or https://url/path.
+	// This must match the "iss" claim in the presented JWT, and the issuer returned from discovery.
+	// Same value as the --oidc-issuer-url flag.
+	// Used to fetch discovery information unless overridden by discoveryURL.
+	// Required to be unique.
+	// Note that egress selection configuration is not used for this network connection.
+	// +required
+	URL string `json:"url"`
+
+	// certificateAuthority contains PEM-encoded certificate authority certificates
+	// used to validate the connection when fetching discovery information.
+	// If unset, the system verifier is used.
+	// Same value as the content of the file referenced by the --oidc-ca-file flag.
+	// +optional
+	CertificateAuthority string `json:"certificateAuthority,omitempty"`
+
+	// audiences is the set of acceptable audiences the JWT must be issued to.
+	// At least one of the entries must match the "aud" claim in presented JWTs.
+	// Same value as the --oidc-client-id flag (though this field supports an array).
+	// Required to be non-empty.
+	// +required
+	Audiences []string `json:"audiences"`
+}
+
+// ClaimValidationRule provides the configuration for a single claim validation rule.
+type ClaimValidationRule struct {
+	// claim is the name of a required claim.
+	// Same as --oidc-required-claim flag.
+	// Only string claim keys are supported.
+	// Mutually exclusive with expression and message.
+	// +optional
+	Claim string `json:"claim,omitempty"`
+	// requiredValue is the value of a required claim.
+	// Same as --oidc-required-claim flag.
+	// Only string claim values are supported.
+	// If claim is set and requiredValue is not set, the claim must be present with a value set to the empty string.
+	// Mutually exclusive with expression and message.
+	// +optional
+	RequiredValue string `json:"requiredValue,omitempty"`
+
+	// expression represents the expression which will be evaluated by CEL.
+	// Must produce a boolean.
+	//
+	// CEL expressions have access to the contents of the token claims, organized into CEL variable:
+	// - 'claims' is a map of claim names to claim values.
+	//   For example, a variable named 'sub' can be accessed as 'claims.sub'.
+	//   Nested claims can be accessed using dot notation, e.g. 'claims.email.verified'.
+	// Must return true for the validation to pass.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	//
+	// Mutually exclusive with claim and requiredValue.
+	// +optional
+	Expression string `json:"expression,omitempty"`
+	// message customizes the returned error message when expression returns false.
+	// message is a literal string.
+	// Mutually exclusive with claim and requiredValue.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// ClaimMappings provides the configuration for claim mapping
+type ClaimMappings struct {
+	// username represents an option for the username attribute.
+	// The claim's value must be a singular string.
+	// Same as the --oidc-username-claim and --oidc-username-prefix flags.
+	// If username.expression is set, the expression must produce a string value.
+	//
+	// In the flag based approach, the --oidc-username-claim and --oidc-username-prefix are optional. If --oidc-username-claim is not set,
+	// the default value is "sub". For the authentication config, there is no defaulting for claim or prefix. The claim and prefix must be set explicitly.
+	// For claim, if --oidc-username-claim was not set with legacy flag approach, configure username.claim="sub" in the authentication config.
+	// For prefix:
+	//     (1) --oidc-username-prefix="-", no prefix was added to the username. For the same behavior using authentication config,
+	//         set username.prefix=""
+	//     (2) --oidc-username-prefix="" and  --oidc-username-claim != "email", prefix was "<value of --oidc-issuer-url>#". For the same
+	//         behavior using authentication config, set username.prefix="<value of issuer.url>#"
+	//	   (3) --oidc-username-prefix="<value>". For the same behavior using authentication config, set username.prefix="<value>"
+	// +required
+	Username PrefixedClaimOrExpression `json:"username"`
+	// groups represents an option for the groups attribute.
+	// The claim's value must be a string or string array claim.
+	// If groups.claim is set, the prefix must be specified (and can be the empty string).
+	// If groups.expression is set, the expression must produce a string or string array value.
+	//  "", [], and null values are treated as the group mapping not being present.
+	// +optional
+	Groups PrefixedClaimOrExpression `json:"groups,omitempty"`
+
+	// uid represents an option for the uid attribute.
+	// Claim must be a singular string claim.
+	// If uid.expression is set, the expression must produce a string value.
+	// +optional
+	UID ClaimOrExpression `json:"uid"`
+
+	// extra represents an option for the extra attribute.
+	// expression must produce a string or string array value.
+	// If the value is empty, the extra mapping will not be present.
+	//
+	// hard-coded extra key/value
+	// - key: "foo"
+	//   valueExpression: "'bar'"
+	// This will result in an extra attribute - foo: ["bar"]
+	//
+	// hard-coded key, value copying claim value
+	// - key: "foo"
+	//   valueExpression: "claims.some_claim"
+	// This will result in an extra attribute - foo: [value of some_claim]
+	//
+	// hard-coded key, value derived from claim value
+	// - key: "admin"
+	//   valueExpression: '(has(claims.is_admin) && claims.is_admin) ? "true":""'
+	// This will result in:
+	//  - if is_admin claim is present and true, extra attribute - admin: ["true"]
+	//  - if is_admin claim is present and false or is_admin claim is not present, no extra attribute will be added
+	//
+	// +optional
+	Extra []ExtraMapping `json:"extra,omitempty"`
+}
+
+// PrefixedClaimOrExpression provides the configuration for a single prefixed claim or expression.
+type PrefixedClaimOrExpression struct {
+	// claim is the JWT claim to use.
+	// Mutually exclusive with expression.
+	// +optional
+	Claim string `json:"claim,omitempty"`
+	// prefix is prepended to claim's value to prevent clashes with existing names.
+	// prefix needs to be set if claim is set and can be the empty string.
+	// Mutually exclusive with expression.
+	// +optional
+	Prefix *string `json:"prefix,omitempty"`
+
+	// expression represents the expression which will be evaluated by CEL.
+	//
+	// CEL expressions have access to the contents of the token claims, organized into CEL variable:
+	// - 'claims' is a map of claim names to claim values.
+	//   For example, a variable named 'sub' can be accessed as 'claims.sub'.
+	//   Nested claims can be accessed using dot notation, e.g. 'claims.email.verified'.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	//
+	// Mutually exclusive with claim and prefix.
+	// +optional
+	Expression string `json:"expression,omitempty"`
+}
+
+// ClaimOrExpression provides the configuration for a single claim or expression.
+type ClaimOrExpression struct {
+	// claim is the JWT claim to use.
+	// Either claim or expression must be set.
+	// Mutually exclusive with expression.
+	// +optional
+	Claim string `json:"claim,omitempty"`
+
+	// expression represents the expression which will be evaluated by CEL.
+	//
+	// CEL expressions have access to the contents of the token claims, organized into CEL variable:
+	// - 'claims' is a map of claim names to claim values.
+	//   For example, a variable named 'sub' can be accessed as 'claims.sub'.
+	//   Nested claims can be accessed using dot notation, e.g. 'claims.email.verified'.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	//
+	// Mutually exclusive with claim.
+	// +optional
+	Expression string `json:"expression,omitempty"`
+}
+
+// ExtraMapping provides the configuration for a single extra mapping.
+type ExtraMapping struct {
+	// key is a string to use as the extra attribute key.
+	// key must be a domain-prefix path (e.g. example.org/foo). All characters before the first "/" must be a valid
+	// subdomain as defined by RFC 1123. All characters trailing the first "/" must
+	// be valid HTTP Path characters as defined by RFC 3986.
+	// key must be lowercase.
+	// +required
+	Key string `json:"key"`
+
+	// valueExpression is a CEL expression to extract extra attribute value.
+	// valueExpression must produce a string or string array value.
+	// "", [], and null values are treated as the extra mapping not being present.
+	// Empty string values contained within a string array are filtered out.
+	//
+	// CEL expressions have access to the contents of the token claims, organized into CEL variable:
+	// - 'claims' is a map of claim names to claim values.
+	//   For example, a variable named 'sub' can be accessed as 'claims.sub'.
+	//   Nested claims can be accessed using dot notation, e.g. 'claims.email.verified'.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	//
+	// +required
+	ValueExpression string `json:"valueExpression"`
+}
+
+// UserValidationRule provides the configuration for a single user info validation rule.
+type UserValidationRule struct {
+	// expression represents the expression which will be evaluated by CEL.
+	// Must return true for the validation to pass.
+	//
+	// CEL expressions have access to the contents of UserInfo, organized into CEL variable:
+	// - 'user' - authentication.k8s.io/v1, Kind=UserInfo object
+	//    Refer to https://github.com/kubernetes/api/blob/release-1.28/authentication/v1/types.go#L105-L122 for the definition.
+	//    API documentation: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#userinfo-v1-authentication-k8s-io
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	//
+	// +required
+	Expression string `json:"expression"`
+
+	// message customizes the returned error message when rule returns false.
+	// message is a literal string.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type AuthorizationConfiguration struct {
+	metav1.TypeMeta
+
+	// Authorizers is an ordered list of authorizers to
+	// authorize requests against.
+	// This is similar to the --authorization-modes kube-apiserver flag
+	// Must be at least one.
+	Authorizers []AuthorizerConfiguration `json:"authorizers"`
+}
+
+const (
+	TypeWebhook                                          AuthorizerType = "Webhook"
+	FailurePolicyNoOpinion                               string         = "NoOpinion"
+	FailurePolicyDeny                                    string         = "Deny"
+	AuthorizationWebhookConnectionInfoTypeKubeConfigFile string         = "KubeConfigFile"
+	AuthorizationWebhookConnectionInfoTypeInCluster      string         = "InClusterConfig"
+)
+
+type AuthorizerType string
+
+type AuthorizerConfiguration struct {
+	// Type refers to the type of the authorizer
+	// "Webhook" is supported in the generic API server
+	// Other API servers may support additional authorizer
+	// types like Node, RBAC, ABAC, etc.
+	Type string `json:"type"`
+
+	// Name used to describe the webhook
+	// This is explicitly used in monitoring machinery for metrics
+	// Note: Names must be DNS1123 labels like `myauthorizername` or
+	//		 subdomains like `myauthorizer.example.domain`
+	// Required, with no default
+	Name string `json:"name"`
+
+	// Webhook defines the configuration for a Webhook authorizer
+	// Must be defined when Type=Webhook
+	// Must not be defined when Type!=Webhook
+	Webhook *WebhookConfiguration `json:"webhook,omitempty"`
+}
+
+type WebhookConfiguration struct {
+	// The duration to cache 'authorized' responses from the webhook
+	// authorizer.
+	// Same as setting `--authorization-webhook-cache-authorized-ttl` flag
+	// Default: 5m0s
+	AuthorizedTTL metav1.Duration `json:"authorizedTTL"`
+	// The duration to cache 'unauthorized' responses from the webhook
+	// authorizer.
+	// Same as setting `--authorization-webhook-cache-unauthorized-ttl` flag
+	// Default: 30s
+	UnauthorizedTTL metav1.Duration `json:"unauthorizedTTL"`
+	// Timeout for the webhook request
+	// Maximum allowed value is 30s.
+	// Required, no default value.
+	Timeout metav1.Duration `json:"timeout"`
+	// The API version of the authorization.k8s.io SubjectAccessReview to
+	// send to and expect from the webhook.
+	// Same as setting `--authorization-webhook-version` flag
+	// Valid values: v1beta1, v1
+	// Required, no default value
+	SubjectAccessReviewVersion string `json:"subjectAccessReviewVersion"`
+	// MatchConditionSubjectAccessReviewVersion specifies the SubjectAccessReview
+	// version the CEL expressions are evaluated against
+	// Valid values: v1
+	// Required, no default value
+	MatchConditionSubjectAccessReviewVersion string `json:"matchConditionSubjectAccessReviewVersion"`
+	// Controls the authorization decision when a webhook request fails to
+	// complete or returns a malformed response or errors evaluating
+	// matchConditions.
+	// Valid values:
+	//   - NoOpinion: continue to subsequent authorizers to see if one of
+	//     them allows the request
+	//   - Deny: reject the request without consulting subsequent authorizers
+	// Required, with no default.
+	FailurePolicy string `json:"failurePolicy"`
+
+	// ConnectionInfo defines how we talk to the webhook
+	ConnectionInfo WebhookConnectionInfo `json:"connectionInfo"`
+
+	// matchConditions is a list of conditions that must be met for a request to be sent to this
+	// webhook. An empty list of matchConditions matches all requests.
+	// There are a maximum of 64 match conditions allowed.
+	//
+	// The exact matching logic is (in order):
+	//   1. If at least one matchCondition evaluates to FALSE, then the webhook is skipped.
+	//   2. If ALL matchConditions evaluate to TRUE, then the webhook is called.
+	//   3. If at least one matchCondition evaluates to an error (but none are FALSE):
+	//      - If failurePolicy=Deny, then the webhook rejects the request
+	//      - If failurePolicy=NoOpinion, then the error is ignored and the webhook is skipped
+	MatchConditions []WebhookMatchCondition `json:"matchConditions"`
+}
+
+type WebhookConnectionInfo struct {
+	// Controls how the webhook should communicate with the server.
+	// Valid values:
+	// - KubeConfigFile: use the file specified in kubeConfigFile to locate the
+	//   server.
+	// - InClusterConfig: use the in-cluster configuration to call the
+	//   SubjectAccessReview API hosted by kube-apiserver. This mode is not
+	//   allowed for kube-apiserver.
+	Type string `json:"type"`
+
+	// Path to KubeConfigFile for connection info
+	// Required, if connectionInfo.Type is KubeConfig
+	KubeConfigFile *string `json:"kubeConfigFile"`
+}
+
+type WebhookMatchCondition struct {
+	// expression represents the expression which will be evaluated by CEL. Must evaluate to bool.
+	// CEL expressions have access to the contents of the SubjectAccessReview in v1 version.
+	// If version specified by subjectAccessReviewVersion in the request variable is v1beta1,
+	// the contents would be converted to the v1 version before evaluating the CEL expression.
+	//
+	// Documentation on CEL: https://kubernetes.io/docs/reference/using-api/cel/
+	Expression string `json:"expression"`
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.conversion.go

@@ -56,6 +56,66 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*AuthenticationConfiguration)(nil), (*apiserver.AuthenticationConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_AuthenticationConfiguration_To_apiserver_AuthenticationConfiguration(a.(*AuthenticationConfiguration), b.(*apiserver.AuthenticationConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.AuthenticationConfiguration)(nil), (*AuthenticationConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_AuthenticationConfiguration_To_v1alpha1_AuthenticationConfiguration(a.(*apiserver.AuthenticationConfiguration), b.(*AuthenticationConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*AuthorizationConfiguration)(nil), (*apiserver.AuthorizationConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_AuthorizationConfiguration_To_apiserver_AuthorizationConfiguration(a.(*AuthorizationConfiguration), b.(*apiserver.AuthorizationConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.AuthorizationConfiguration)(nil), (*AuthorizationConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_AuthorizationConfiguration_To_v1alpha1_AuthorizationConfiguration(a.(*apiserver.AuthorizationConfiguration), b.(*AuthorizationConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*AuthorizerConfiguration)(nil), (*apiserver.AuthorizerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_AuthorizerConfiguration_To_apiserver_AuthorizerConfiguration(a.(*AuthorizerConfiguration), b.(*apiserver.AuthorizerConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.AuthorizerConfiguration)(nil), (*AuthorizerConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_AuthorizerConfiguration_To_v1alpha1_AuthorizerConfiguration(a.(*apiserver.AuthorizerConfiguration), b.(*AuthorizerConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*ClaimMappings)(nil), (*apiserver.ClaimMappings)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings(a.(*ClaimMappings), b.(*apiserver.ClaimMappings), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.ClaimMappings)(nil), (*ClaimMappings)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings(a.(*apiserver.ClaimMappings), b.(*ClaimMappings), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*ClaimOrExpression)(nil), (*apiserver.ClaimOrExpression)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression(a.(*ClaimOrExpression), b.(*apiserver.ClaimOrExpression), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.ClaimOrExpression)(nil), (*ClaimOrExpression)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression(a.(*apiserver.ClaimOrExpression), b.(*ClaimOrExpression), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*ClaimValidationRule)(nil), (*apiserver.ClaimValidationRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_ClaimValidationRule_To_apiserver_ClaimValidationRule(a.(*ClaimValidationRule), b.(*apiserver.ClaimValidationRule), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.ClaimValidationRule)(nil), (*ClaimValidationRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_ClaimValidationRule_To_v1alpha1_ClaimValidationRule(a.(*apiserver.ClaimValidationRule), b.(*ClaimValidationRule), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*Connection)(nil), (*apiserver.Connection)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_Connection_To_apiserver_Connection(a.(*Connection), b.(*apiserver.Connection), scope)
 	}); err != nil {
@@ -81,6 +141,46 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*ExtraMapping)(nil), (*apiserver.ExtraMapping)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_ExtraMapping_To_apiserver_ExtraMapping(a.(*ExtraMapping), b.(*apiserver.ExtraMapping), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.ExtraMapping)(nil), (*ExtraMapping)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_ExtraMapping_To_v1alpha1_ExtraMapping(a.(*apiserver.ExtraMapping), b.(*ExtraMapping), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*Issuer)(nil), (*apiserver.Issuer)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_Issuer_To_apiserver_Issuer(a.(*Issuer), b.(*apiserver.Issuer), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.Issuer)(nil), (*Issuer)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_Issuer_To_v1alpha1_Issuer(a.(*apiserver.Issuer), b.(*Issuer), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*JWTAuthenticator)(nil), (*apiserver.JWTAuthenticator)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_JWTAuthenticator_To_apiserver_JWTAuthenticator(a.(*JWTAuthenticator), b.(*apiserver.JWTAuthenticator), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.JWTAuthenticator)(nil), (*JWTAuthenticator)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_JWTAuthenticator_To_v1alpha1_JWTAuthenticator(a.(*apiserver.JWTAuthenticator), b.(*JWTAuthenticator), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*PrefixedClaimOrExpression)(nil), (*apiserver.PrefixedClaimOrExpression)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(a.(*PrefixedClaimOrExpression), b.(*apiserver.PrefixedClaimOrExpression), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.PrefixedClaimOrExpression)(nil), (*PrefixedClaimOrExpression)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(a.(*apiserver.PrefixedClaimOrExpression), b.(*PrefixedClaimOrExpression), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*TCPTransport)(nil), (*apiserver.TCPTransport)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(a.(*TCPTransport), b.(*apiserver.TCPTransport), scope)
 	}); err != nil {
@@ -131,6 +231,46 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*UserValidationRule)(nil), (*apiserver.UserValidationRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_UserValidationRule_To_apiserver_UserValidationRule(a.(*UserValidationRule), b.(*apiserver.UserValidationRule), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.UserValidationRule)(nil), (*UserValidationRule)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_UserValidationRule_To_v1alpha1_UserValidationRule(a.(*apiserver.UserValidationRule), b.(*UserValidationRule), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*WebhookConfiguration)(nil), (*apiserver.WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_WebhookConfiguration_To_apiserver_WebhookConfiguration(a.(*WebhookConfiguration), b.(*apiserver.WebhookConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.WebhookConfiguration)(nil), (*WebhookConfiguration)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(a.(*apiserver.WebhookConfiguration), b.(*WebhookConfiguration), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*WebhookConnectionInfo)(nil), (*apiserver.WebhookConnectionInfo)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo(a.(*WebhookConnectionInfo), b.(*apiserver.WebhookConnectionInfo), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.WebhookConnectionInfo)(nil), (*WebhookConnectionInfo)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo(a.(*apiserver.WebhookConnectionInfo), b.(*WebhookConnectionInfo), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*WebhookMatchCondition)(nil), (*apiserver.WebhookMatchCondition)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha1_WebhookMatchCondition_To_apiserver_WebhookMatchCondition(a.(*WebhookMatchCondition), b.(*apiserver.WebhookMatchCondition), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*apiserver.WebhookMatchCondition)(nil), (*WebhookMatchCondition)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_apiserver_WebhookMatchCondition_To_v1alpha1_WebhookMatchCondition(a.(*apiserver.WebhookMatchCondition), b.(*WebhookMatchCondition), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddConversionFunc((*EgressSelection)(nil), (*apiserver.EgressSelection)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha1_EgressSelection_To_apiserver_EgressSelection(a.(*EgressSelection), b.(*apiserver.EgressSelection), scope)
 	}); err != nil {
@@ -183,6 +323,156 @@ func Convert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginC
 	return autoConvert_apiserver_AdmissionPluginConfiguration_To_v1alpha1_AdmissionPluginConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha1_AuthenticationConfiguration_To_apiserver_AuthenticationConfiguration(in *AuthenticationConfiguration, out *apiserver.AuthenticationConfiguration, s conversion.Scope) error {
+	out.JWT = *(*[]apiserver.JWTAuthenticator)(unsafe.Pointer(&in.JWT))
+	return nil
+}
+
+// Convert_v1alpha1_AuthenticationConfiguration_To_apiserver_AuthenticationConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_AuthenticationConfiguration_To_apiserver_AuthenticationConfiguration(in *AuthenticationConfiguration, out *apiserver.AuthenticationConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AuthenticationConfiguration_To_apiserver_AuthenticationConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_AuthenticationConfiguration_To_v1alpha1_AuthenticationConfiguration(in *apiserver.AuthenticationConfiguration, out *AuthenticationConfiguration, s conversion.Scope) error {
+	out.JWT = *(*[]JWTAuthenticator)(unsafe.Pointer(&in.JWT))
+	return nil
+}
+
+// Convert_apiserver_AuthenticationConfiguration_To_v1alpha1_AuthenticationConfiguration is an autogenerated conversion function.
+func Convert_apiserver_AuthenticationConfiguration_To_v1alpha1_AuthenticationConfiguration(in *apiserver.AuthenticationConfiguration, out *AuthenticationConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_AuthenticationConfiguration_To_v1alpha1_AuthenticationConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_AuthorizationConfiguration_To_apiserver_AuthorizationConfiguration(in *AuthorizationConfiguration, out *apiserver.AuthorizationConfiguration, s conversion.Scope) error {
+	out.Authorizers = *(*[]apiserver.AuthorizerConfiguration)(unsafe.Pointer(&in.Authorizers))
+	return nil
+}
+
+// Convert_v1alpha1_AuthorizationConfiguration_To_apiserver_AuthorizationConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_AuthorizationConfiguration_To_apiserver_AuthorizationConfiguration(in *AuthorizationConfiguration, out *apiserver.AuthorizationConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AuthorizationConfiguration_To_apiserver_AuthorizationConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_AuthorizationConfiguration_To_v1alpha1_AuthorizationConfiguration(in *apiserver.AuthorizationConfiguration, out *AuthorizationConfiguration, s conversion.Scope) error {
+	out.Authorizers = *(*[]AuthorizerConfiguration)(unsafe.Pointer(&in.Authorizers))
+	return nil
+}
+
+// Convert_apiserver_AuthorizationConfiguration_To_v1alpha1_AuthorizationConfiguration is an autogenerated conversion function.
+func Convert_apiserver_AuthorizationConfiguration_To_v1alpha1_AuthorizationConfiguration(in *apiserver.AuthorizationConfiguration, out *AuthorizationConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_AuthorizationConfiguration_To_v1alpha1_AuthorizationConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_AuthorizerConfiguration_To_apiserver_AuthorizerConfiguration(in *AuthorizerConfiguration, out *apiserver.AuthorizerConfiguration, s conversion.Scope) error {
+	out.Type = apiserver.AuthorizerType(in.Type)
+	out.Name = in.Name
+	out.Webhook = (*apiserver.WebhookConfiguration)(unsafe.Pointer(in.Webhook))
+	return nil
+}
+
+// Convert_v1alpha1_AuthorizerConfiguration_To_apiserver_AuthorizerConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_AuthorizerConfiguration_To_apiserver_AuthorizerConfiguration(in *AuthorizerConfiguration, out *apiserver.AuthorizerConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_AuthorizerConfiguration_To_apiserver_AuthorizerConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_AuthorizerConfiguration_To_v1alpha1_AuthorizerConfiguration(in *apiserver.AuthorizerConfiguration, out *AuthorizerConfiguration, s conversion.Scope) error {
+	out.Type = string(in.Type)
+	out.Name = in.Name
+	out.Webhook = (*WebhookConfiguration)(unsafe.Pointer(in.Webhook))
+	return nil
+}
+
+// Convert_apiserver_AuthorizerConfiguration_To_v1alpha1_AuthorizerConfiguration is an autogenerated conversion function.
+func Convert_apiserver_AuthorizerConfiguration_To_v1alpha1_AuthorizerConfiguration(in *apiserver.AuthorizerConfiguration, out *AuthorizerConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_AuthorizerConfiguration_To_v1alpha1_AuthorizerConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings(in *ClaimMappings, out *apiserver.ClaimMappings, s conversion.Scope) error {
+	if err := Convert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(&in.Username, &out.Username, s); err != nil {
+		return err
+	}
+	if err := Convert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(&in.Groups, &out.Groups, s); err != nil {
+		return err
+	}
+	if err := Convert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression(&in.UID, &out.UID, s); err != nil {
+		return err
+	}
+	out.Extra = *(*[]apiserver.ExtraMapping)(unsafe.Pointer(&in.Extra))
+	return nil
+}
+
+// Convert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings is an autogenerated conversion function.
+func Convert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings(in *ClaimMappings, out *apiserver.ClaimMappings, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings(in, out, s)
+}
+
+func autoConvert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings(in *apiserver.ClaimMappings, out *ClaimMappings, s conversion.Scope) error {
+	if err := Convert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(&in.Username, &out.Username, s); err != nil {
+		return err
+	}
+	if err := Convert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(&in.Groups, &out.Groups, s); err != nil {
+		return err
+	}
+	if err := Convert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression(&in.UID, &out.UID, s); err != nil {
+		return err
+	}
+	out.Extra = *(*[]ExtraMapping)(unsafe.Pointer(&in.Extra))
+	return nil
+}
+
+// Convert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings is an autogenerated conversion function.
+func Convert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings(in *apiserver.ClaimMappings, out *ClaimMappings, s conversion.Scope) error {
+	return autoConvert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings(in, out, s)
+}
+
+func autoConvert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression(in *ClaimOrExpression, out *apiserver.ClaimOrExpression, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression is an autogenerated conversion function.
+func Convert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression(in *ClaimOrExpression, out *apiserver.ClaimOrExpression, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ClaimOrExpression_To_apiserver_ClaimOrExpression(in, out, s)
+}
+
+func autoConvert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression(in *apiserver.ClaimOrExpression, out *ClaimOrExpression, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression is an autogenerated conversion function.
+func Convert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression(in *apiserver.ClaimOrExpression, out *ClaimOrExpression, s conversion.Scope) error {
+	return autoConvert_apiserver_ClaimOrExpression_To_v1alpha1_ClaimOrExpression(in, out, s)
+}
+
+func autoConvert_v1alpha1_ClaimValidationRule_To_apiserver_ClaimValidationRule(in *ClaimValidationRule, out *apiserver.ClaimValidationRule, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.RequiredValue = in.RequiredValue
+	out.Expression = in.Expression
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_v1alpha1_ClaimValidationRule_To_apiserver_ClaimValidationRule is an autogenerated conversion function.
+func Convert_v1alpha1_ClaimValidationRule_To_apiserver_ClaimValidationRule(in *ClaimValidationRule, out *apiserver.ClaimValidationRule, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ClaimValidationRule_To_apiserver_ClaimValidationRule(in, out, s)
+}
+
+func autoConvert_apiserver_ClaimValidationRule_To_v1alpha1_ClaimValidationRule(in *apiserver.ClaimValidationRule, out *ClaimValidationRule, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.RequiredValue = in.RequiredValue
+	out.Expression = in.Expression
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_apiserver_ClaimValidationRule_To_v1alpha1_ClaimValidationRule is an autogenerated conversion function.
+func Convert_apiserver_ClaimValidationRule_To_v1alpha1_ClaimValidationRule(in *apiserver.ClaimValidationRule, out *ClaimValidationRule, s conversion.Scope) error {
+	return autoConvert_apiserver_ClaimValidationRule_To_v1alpha1_ClaimValidationRule(in, out, s)
+}
+
 func autoConvert_v1alpha1_Connection_To_apiserver_Connection(in *Connection, out *apiserver.Connection, s conversion.Scope) error {
 	out.ProxyProtocol = apiserver.ProtocolType(in.ProxyProtocol)
 	out.Transport = (*apiserver.Transport)(unsafe.Pointer(in.Transport))
@@ -266,6 +556,110 @@ func Convert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorCon
 	return autoConvert_apiserver_EgressSelectorConfiguration_To_v1alpha1_EgressSelectorConfiguration(in, out, s)
 }
 
+func autoConvert_v1alpha1_ExtraMapping_To_apiserver_ExtraMapping(in *ExtraMapping, out *apiserver.ExtraMapping, s conversion.Scope) error {
+	out.Key = in.Key
+	out.ValueExpression = in.ValueExpression
+	return nil
+}
+
+// Convert_v1alpha1_ExtraMapping_To_apiserver_ExtraMapping is an autogenerated conversion function.
+func Convert_v1alpha1_ExtraMapping_To_apiserver_ExtraMapping(in *ExtraMapping, out *apiserver.ExtraMapping, s conversion.Scope) error {
+	return autoConvert_v1alpha1_ExtraMapping_To_apiserver_ExtraMapping(in, out, s)
+}
+
+func autoConvert_apiserver_ExtraMapping_To_v1alpha1_ExtraMapping(in *apiserver.ExtraMapping, out *ExtraMapping, s conversion.Scope) error {
+	out.Key = in.Key
+	out.ValueExpression = in.ValueExpression
+	return nil
+}
+
+// Convert_apiserver_ExtraMapping_To_v1alpha1_ExtraMapping is an autogenerated conversion function.
+func Convert_apiserver_ExtraMapping_To_v1alpha1_ExtraMapping(in *apiserver.ExtraMapping, out *ExtraMapping, s conversion.Scope) error {
+	return autoConvert_apiserver_ExtraMapping_To_v1alpha1_ExtraMapping(in, out, s)
+}
+
+func autoConvert_v1alpha1_Issuer_To_apiserver_Issuer(in *Issuer, out *apiserver.Issuer, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CertificateAuthority = in.CertificateAuthority
+	out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences))
+	return nil
+}
+
+// Convert_v1alpha1_Issuer_To_apiserver_Issuer is an autogenerated conversion function.
+func Convert_v1alpha1_Issuer_To_apiserver_Issuer(in *Issuer, out *apiserver.Issuer, s conversion.Scope) error {
+	return autoConvert_v1alpha1_Issuer_To_apiserver_Issuer(in, out, s)
+}
+
+func autoConvert_apiserver_Issuer_To_v1alpha1_Issuer(in *apiserver.Issuer, out *Issuer, s conversion.Scope) error {
+	out.URL = in.URL
+	out.CertificateAuthority = in.CertificateAuthority
+	out.Audiences = *(*[]string)(unsafe.Pointer(&in.Audiences))
+	return nil
+}
+
+// Convert_apiserver_Issuer_To_v1alpha1_Issuer is an autogenerated conversion function.
+func Convert_apiserver_Issuer_To_v1alpha1_Issuer(in *apiserver.Issuer, out *Issuer, s conversion.Scope) error {
+	return autoConvert_apiserver_Issuer_To_v1alpha1_Issuer(in, out, s)
+}
+
+func autoConvert_v1alpha1_JWTAuthenticator_To_apiserver_JWTAuthenticator(in *JWTAuthenticator, out *apiserver.JWTAuthenticator, s conversion.Scope) error {
+	if err := Convert_v1alpha1_Issuer_To_apiserver_Issuer(&in.Issuer, &out.Issuer, s); err != nil {
+		return err
+	}
+	out.ClaimValidationRules = *(*[]apiserver.ClaimValidationRule)(unsafe.Pointer(&in.ClaimValidationRules))
+	if err := Convert_v1alpha1_ClaimMappings_To_apiserver_ClaimMappings(&in.ClaimMappings, &out.ClaimMappings, s); err != nil {
+		return err
+	}
+	out.UserValidationRules = *(*[]apiserver.UserValidationRule)(unsafe.Pointer(&in.UserValidationRules))
+	return nil
+}
+
+// Convert_v1alpha1_JWTAuthenticator_To_apiserver_JWTAuthenticator is an autogenerated conversion function.
+func Convert_v1alpha1_JWTAuthenticator_To_apiserver_JWTAuthenticator(in *JWTAuthenticator, out *apiserver.JWTAuthenticator, s conversion.Scope) error {
+	return autoConvert_v1alpha1_JWTAuthenticator_To_apiserver_JWTAuthenticator(in, out, s)
+}
+
+func autoConvert_apiserver_JWTAuthenticator_To_v1alpha1_JWTAuthenticator(in *apiserver.JWTAuthenticator, out *JWTAuthenticator, s conversion.Scope) error {
+	if err := Convert_apiserver_Issuer_To_v1alpha1_Issuer(&in.Issuer, &out.Issuer, s); err != nil {
+		return err
+	}
+	out.ClaimValidationRules = *(*[]ClaimValidationRule)(unsafe.Pointer(&in.ClaimValidationRules))
+	if err := Convert_apiserver_ClaimMappings_To_v1alpha1_ClaimMappings(&in.ClaimMappings, &out.ClaimMappings, s); err != nil {
+		return err
+	}
+	out.UserValidationRules = *(*[]UserValidationRule)(unsafe.Pointer(&in.UserValidationRules))
+	return nil
+}
+
+// Convert_apiserver_JWTAuthenticator_To_v1alpha1_JWTAuthenticator is an autogenerated conversion function.
+func Convert_apiserver_JWTAuthenticator_To_v1alpha1_JWTAuthenticator(in *apiserver.JWTAuthenticator, out *JWTAuthenticator, s conversion.Scope) error {
+	return autoConvert_apiserver_JWTAuthenticator_To_v1alpha1_JWTAuthenticator(in, out, s)
+}
+
+func autoConvert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(in *PrefixedClaimOrExpression, out *apiserver.PrefixedClaimOrExpression, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.Prefix = (*string)(unsafe.Pointer(in.Prefix))
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression is an autogenerated conversion function.
+func Convert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(in *PrefixedClaimOrExpression, out *apiserver.PrefixedClaimOrExpression, s conversion.Scope) error {
+	return autoConvert_v1alpha1_PrefixedClaimOrExpression_To_apiserver_PrefixedClaimOrExpression(in, out, s)
+}
+
+func autoConvert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(in *apiserver.PrefixedClaimOrExpression, out *PrefixedClaimOrExpression, s conversion.Scope) error {
+	out.Claim = in.Claim
+	out.Prefix = (*string)(unsafe.Pointer(in.Prefix))
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression is an autogenerated conversion function.
+func Convert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(in *apiserver.PrefixedClaimOrExpression, out *PrefixedClaimOrExpression, s conversion.Scope) error {
+	return autoConvert_apiserver_PrefixedClaimOrExpression_To_v1alpha1_PrefixedClaimOrExpression(in, out, s)
+}
+
 func autoConvert_v1alpha1_TCPTransport_To_apiserver_TCPTransport(in *TCPTransport, out *apiserver.TCPTransport, s conversion.Scope) error {
 	out.URL = in.URL
 	out.TLSConfig = (*apiserver.TLSConfig)(unsafe.Pointer(in.TLSConfig))
@@ -373,3 +767,105 @@ func autoConvert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in *apiserver.U
 func Convert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in *apiserver.UDSTransport, out *UDSTransport, s conversion.Scope) error {
 	return autoConvert_apiserver_UDSTransport_To_v1alpha1_UDSTransport(in, out, s)
 }
+
+func autoConvert_v1alpha1_UserValidationRule_To_apiserver_UserValidationRule(in *UserValidationRule, out *apiserver.UserValidationRule, s conversion.Scope) error {
+	out.Expression = in.Expression
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_v1alpha1_UserValidationRule_To_apiserver_UserValidationRule is an autogenerated conversion function.
+func Convert_v1alpha1_UserValidationRule_To_apiserver_UserValidationRule(in *UserValidationRule, out *apiserver.UserValidationRule, s conversion.Scope) error {
+	return autoConvert_v1alpha1_UserValidationRule_To_apiserver_UserValidationRule(in, out, s)
+}
+
+func autoConvert_apiserver_UserValidationRule_To_v1alpha1_UserValidationRule(in *apiserver.UserValidationRule, out *UserValidationRule, s conversion.Scope) error {
+	out.Expression = in.Expression
+	out.Message = in.Message
+	return nil
+}
+
+// Convert_apiserver_UserValidationRule_To_v1alpha1_UserValidationRule is an autogenerated conversion function.
+func Convert_apiserver_UserValidationRule_To_v1alpha1_UserValidationRule(in *apiserver.UserValidationRule, out *UserValidationRule, s conversion.Scope) error {
+	return autoConvert_apiserver_UserValidationRule_To_v1alpha1_UserValidationRule(in, out, s)
+}
+
+func autoConvert_v1alpha1_WebhookConfiguration_To_apiserver_WebhookConfiguration(in *WebhookConfiguration, out *apiserver.WebhookConfiguration, s conversion.Scope) error {
+	out.AuthorizedTTL = in.AuthorizedTTL
+	out.UnauthorizedTTL = in.UnauthorizedTTL
+	out.Timeout = in.Timeout
+	out.SubjectAccessReviewVersion = in.SubjectAccessReviewVersion
+	out.MatchConditionSubjectAccessReviewVersion = in.MatchConditionSubjectAccessReviewVersion
+	out.FailurePolicy = in.FailurePolicy
+	if err := Convert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo(&in.ConnectionInfo, &out.ConnectionInfo, s); err != nil {
+		return err
+	}
+	out.MatchConditions = *(*[]apiserver.WebhookMatchCondition)(unsafe.Pointer(&in.MatchConditions))
+	return nil
+}
+
+// Convert_v1alpha1_WebhookConfiguration_To_apiserver_WebhookConfiguration is an autogenerated conversion function.
+func Convert_v1alpha1_WebhookConfiguration_To_apiserver_WebhookConfiguration(in *WebhookConfiguration, out *apiserver.WebhookConfiguration, s conversion.Scope) error {
+	return autoConvert_v1alpha1_WebhookConfiguration_To_apiserver_WebhookConfiguration(in, out, s)
+}
+
+func autoConvert_apiserver_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *apiserver.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error {
+	out.AuthorizedTTL = in.AuthorizedTTL
+	out.UnauthorizedTTL = in.UnauthorizedTTL
+	out.Timeout = in.Timeout
+	out.SubjectAccessReviewVersion = in.SubjectAccessReviewVersion
+	out.MatchConditionSubjectAccessReviewVersion = in.MatchConditionSubjectAccessReviewVersion
+	out.FailurePolicy = in.FailurePolicy
+	if err := Convert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo(&in.ConnectionInfo, &out.ConnectionInfo, s); err != nil {
+		return err
+	}
+	out.MatchConditions = *(*[]WebhookMatchCondition)(unsafe.Pointer(&in.MatchConditions))
+	return nil
+}
+
+// Convert_apiserver_WebhookConfiguration_To_v1alpha1_WebhookConfiguration is an autogenerated conversion function.
+func Convert_apiserver_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in *apiserver.WebhookConfiguration, out *WebhookConfiguration, s conversion.Scope) error {
+	return autoConvert_apiserver_WebhookConfiguration_To_v1alpha1_WebhookConfiguration(in, out, s)
+}
+
+func autoConvert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo(in *WebhookConnectionInfo, out *apiserver.WebhookConnectionInfo, s conversion.Scope) error {
+	out.Type = in.Type
+	out.KubeConfigFile = (*string)(unsafe.Pointer(in.KubeConfigFile))
+	return nil
+}
+
+// Convert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo is an autogenerated conversion function.
+func Convert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo(in *WebhookConnectionInfo, out *apiserver.WebhookConnectionInfo, s conversion.Scope) error {
+	return autoConvert_v1alpha1_WebhookConnectionInfo_To_apiserver_WebhookConnectionInfo(in, out, s)
+}
+
+func autoConvert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo(in *apiserver.WebhookConnectionInfo, out *WebhookConnectionInfo, s conversion.Scope) error {
+	out.Type = in.Type
+	out.KubeConfigFile = (*string)(unsafe.Pointer(in.KubeConfigFile))
+	return nil
+}
+
+// Convert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo is an autogenerated conversion function.
+func Convert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo(in *apiserver.WebhookConnectionInfo, out *WebhookConnectionInfo, s conversion.Scope) error {
+	return autoConvert_apiserver_WebhookConnectionInfo_To_v1alpha1_WebhookConnectionInfo(in, out, s)
+}
+
+func autoConvert_v1alpha1_WebhookMatchCondition_To_apiserver_WebhookMatchCondition(in *WebhookMatchCondition, out *apiserver.WebhookMatchCondition, s conversion.Scope) error {
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_v1alpha1_WebhookMatchCondition_To_apiserver_WebhookMatchCondition is an autogenerated conversion function.
+func Convert_v1alpha1_WebhookMatchCondition_To_apiserver_WebhookMatchCondition(in *WebhookMatchCondition, out *apiserver.WebhookMatchCondition, s conversion.Scope) error {
+	return autoConvert_v1alpha1_WebhookMatchCondition_To_apiserver_WebhookMatchCondition(in, out, s)
+}
+
+func autoConvert_apiserver_WebhookMatchCondition_To_v1alpha1_WebhookMatchCondition(in *apiserver.WebhookMatchCondition, out *WebhookMatchCondition, s conversion.Scope) error {
+	out.Expression = in.Expression
+	return nil
+}
+
+// Convert_apiserver_WebhookMatchCondition_To_v1alpha1_WebhookMatchCondition is an autogenerated conversion function.
+func Convert_apiserver_WebhookMatchCondition_To_v1alpha1_WebhookMatchCondition(in *apiserver.WebhookMatchCondition, out *WebhookMatchCondition, s conversion.Scope) error {
+	return autoConvert_apiserver_WebhookMatchCondition_To_v1alpha1_WebhookMatchCondition(in, out, s)
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.deepcopy.go

@@ -78,6 +78,147 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthenticationConfiguration) DeepCopyInto(out *AuthenticationConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.JWT != nil {
+		in, out := &in.JWT, &out.JWT
+		*out = make([]JWTAuthenticator, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthenticationConfiguration.
+func (in *AuthenticationConfiguration) DeepCopy() *AuthenticationConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthenticationConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AuthenticationConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizationConfiguration) DeepCopyInto(out *AuthorizationConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.Authorizers != nil {
+		in, out := &in.Authorizers, &out.Authorizers
+		*out = make([]AuthorizerConfiguration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationConfiguration.
+func (in *AuthorizationConfiguration) DeepCopy() *AuthorizationConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizationConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AuthorizationConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizerConfiguration) DeepCopyInto(out *AuthorizerConfiguration) {
+	*out = *in
+	if in.Webhook != nil {
+		in, out := &in.Webhook, &out.Webhook
+		*out = new(WebhookConfiguration)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizerConfiguration.
+func (in *AuthorizerConfiguration) DeepCopy() *AuthorizerConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizerConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimMappings) DeepCopyInto(out *ClaimMappings) {
+	*out = *in
+	in.Username.DeepCopyInto(&out.Username)
+	in.Groups.DeepCopyInto(&out.Groups)
+	out.UID = in.UID
+	if in.Extra != nil {
+		in, out := &in.Extra, &out.Extra
+		*out = make([]ExtraMapping, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimMappings.
+func (in *ClaimMappings) DeepCopy() *ClaimMappings {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimMappings)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimOrExpression) DeepCopyInto(out *ClaimOrExpression) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimOrExpression.
+func (in *ClaimOrExpression) DeepCopy() *ClaimOrExpression {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimOrExpression)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimValidationRule) DeepCopyInto(out *ClaimValidationRule) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimValidationRule.
+func (in *ClaimValidationRule) DeepCopy() *ClaimValidationRule {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimValidationRule)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Connection) DeepCopyInto(out *Connection) {
 	*out = *in
@@ -148,6 +289,92 @@ func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExtraMapping) DeepCopyInto(out *ExtraMapping) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraMapping.
+func (in *ExtraMapping) DeepCopy() *ExtraMapping {
+	if in == nil {
+		return nil
+	}
+	out := new(ExtraMapping)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Issuer) DeepCopyInto(out *Issuer) {
+	*out = *in
+	if in.Audiences != nil {
+		in, out := &in.Audiences, &out.Audiences
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Issuer.
+func (in *Issuer) DeepCopy() *Issuer {
+	if in == nil {
+		return nil
+	}
+	out := new(Issuer)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JWTAuthenticator) DeepCopyInto(out *JWTAuthenticator) {
+	*out = *in
+	in.Issuer.DeepCopyInto(&out.Issuer)
+	if in.ClaimValidationRules != nil {
+		in, out := &in.ClaimValidationRules, &out.ClaimValidationRules
+		*out = make([]ClaimValidationRule, len(*in))
+		copy(*out, *in)
+	}
+	in.ClaimMappings.DeepCopyInto(&out.ClaimMappings)
+	if in.UserValidationRules != nil {
+		in, out := &in.UserValidationRules, &out.UserValidationRules
+		*out = make([]UserValidationRule, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthenticator.
+func (in *JWTAuthenticator) DeepCopy() *JWTAuthenticator {
+	if in == nil {
+		return nil
+	}
+	out := new(JWTAuthenticator)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrefixedClaimOrExpression) DeepCopyInto(out *PrefixedClaimOrExpression) {
+	*out = *in
+	if in.Prefix != nil {
+		in, out := &in.Prefix, &out.Prefix
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrefixedClaimOrExpression.
+func (in *PrefixedClaimOrExpression) DeepCopy() *PrefixedClaimOrExpression {
+	if in == nil {
+		return nil
+	}
+	out := new(PrefixedClaimOrExpression)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TCPTransport) DeepCopyInto(out *TCPTransport) {
 	*out = *in
@@ -252,3 +479,81 @@ func (in *UDSTransport) DeepCopy() *UDSTransport {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UserValidationRule) DeepCopyInto(out *UserValidationRule) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserValidationRule.
+func (in *UserValidationRule) DeepCopy() *UserValidationRule {
+	if in == nil {
+		return nil
+	}
+	out := new(UserValidationRule)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) {
+	*out = *in
+	out.AuthorizedTTL = in.AuthorizedTTL
+	out.UnauthorizedTTL = in.UnauthorizedTTL
+	out.Timeout = in.Timeout
+	in.ConnectionInfo.DeepCopyInto(&out.ConnectionInfo)
+	if in.MatchConditions != nil {
+		in, out := &in.MatchConditions, &out.MatchConditions
+		*out = make([]WebhookMatchCondition, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration.
+func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookConnectionInfo) DeepCopyInto(out *WebhookConnectionInfo) {
+	*out = *in
+	if in.KubeConfigFile != nil {
+		in, out := &in.KubeConfigFile, &out.KubeConfigFile
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConnectionInfo.
+func (in *WebhookConnectionInfo) DeepCopy() *WebhookConnectionInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookConnectionInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookMatchCondition) DeepCopyInto(out *WebhookMatchCondition) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookMatchCondition.
+func (in *WebhookMatchCondition) DeepCopy() *WebhookMatchCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookMatchCondition)
+	in.DeepCopyInto(out)
+	return out
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/zz_generated.defaults.go

@@ -29,5 +29,15 @@ import (
 // Public to allow building arbitrary schemes.
 // All generated defaulters are covering - they call all nested defaulters.
 func RegisterDefaults(scheme *runtime.Scheme) error {
+	scheme.AddTypeDefaultingFunc(&AuthorizationConfiguration{}, func(obj interface{}) { SetObjectDefaults_AuthorizationConfiguration(obj.(*AuthorizationConfiguration)) })
 	return nil
 }
+
+func SetObjectDefaults_AuthorizationConfiguration(in *AuthorizationConfiguration) {
+	for i := range in.Authorizers {
+		a := &in.Authorizers[i]
+		if a.Webhook != nil {
+			SetDefaults_WebhookConfiguration(a.Webhook)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/validation/validation.go

@@ -0,0 +1,630 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validation
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	v1 "k8s.io/api/authorization/v1"
+	"k8s.io/api/authorization/v1beta1"
+	"k8s.io/apimachinery/pkg/util/sets"
+	utilvalidation "k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	api "k8s.io/apiserver/pkg/apis/apiserver"
+	authenticationcel "k8s.io/apiserver/pkg/authentication/cel"
+	authorizationcel "k8s.io/apiserver/pkg/authorization/cel"
+	"k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
+	"k8s.io/apiserver/pkg/features"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/util/cert"
+)
+
+const (
+	atLeastOneRequiredErrFmt = "at least one %s is required"
+)
+
+var (
+	root = field.NewPath("jwt")
+)
+
+// ValidateAuthenticationConfiguration validates a given AuthenticationConfiguration.
+func ValidateAuthenticationConfiguration(c *api.AuthenticationConfiguration) field.ErrorList {
+	var allErrs field.ErrorList
+
+	// This stricter validation is solely based on what the current implementation supports.
+	// TODO(aramase): when StructuredAuthenticationConfiguration feature gate is added and wired up,
+	// relax this check to allow 0 authenticators. This will allow us to support the case where
+	// API server is initially configured with no authenticators and then authenticators are added
+	// later via dynamic config.
+	if len(c.JWT) == 0 {
+		allErrs = append(allErrs, field.Required(root, fmt.Sprintf(atLeastOneRequiredErrFmt, root)))
+		return allErrs
+	}
+
+	// This stricter validation is because the --oidc-* flag option is singular.
+	// TODO(aramase): when StructuredAuthenticationConfiguration feature gate is added and wired up,
+	// remove the 1 authenticator limit check and add set the limit to 64.
+	if len(c.JWT) > 1 {
+		allErrs = append(allErrs, field.TooMany(root, len(c.JWT), 1))
+		return allErrs
+	}
+
+	// TODO(aramase): right now we only support a single JWT authenticator as
+	// this is wired to the --oidc-* flags. When StructuredAuthenticationConfiguration
+	// feature gate is added and wired up, we will remove the 1 authenticator limit
+	// check and add validation for duplicate issuers.
+	for i, a := range c.JWT {
+		fldPath := root.Index(i)
+		_, errs := validateJWTAuthenticator(a, fldPath, utilfeature.DefaultFeatureGate.Enabled(features.StructuredAuthenticationConfiguration))
+		allErrs = append(allErrs, errs...)
+	}
+
+	return allErrs
+}
+
+// CompileAndValidateJWTAuthenticator validates a given JWTAuthenticator and returns a CELMapper with the compiled
+// CEL expressions for claim mappings and validation rules.
+// This is exported for use in oidc package.
+func CompileAndValidateJWTAuthenticator(authenticator api.JWTAuthenticator) (authenticationcel.CELMapper, field.ErrorList) {
+	return validateJWTAuthenticator(authenticator, nil, utilfeature.DefaultFeatureGate.Enabled(features.StructuredAuthenticationConfiguration))
+}
+
+func validateJWTAuthenticator(authenticator api.JWTAuthenticator, fldPath *field.Path, structuredAuthnFeatureEnabled bool) (authenticationcel.CELMapper, field.ErrorList) {
+	var allErrs field.ErrorList
+
+	compiler := authenticationcel.NewCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
+	mapper := &authenticationcel.CELMapper{}
+
+	allErrs = append(allErrs, validateIssuer(authenticator.Issuer, fldPath.Child("issuer"))...)
+	allErrs = append(allErrs, validateClaimValidationRules(compiler, mapper, authenticator.ClaimValidationRules, fldPath.Child("claimValidationRules"), structuredAuthnFeatureEnabled)...)
+	allErrs = append(allErrs, validateClaimMappings(compiler, mapper, authenticator.ClaimMappings, fldPath.Child("claimMappings"), structuredAuthnFeatureEnabled)...)
+	allErrs = append(allErrs, validateUserValidationRules(compiler, mapper, authenticator.UserValidationRules, fldPath.Child("userValidationRules"), structuredAuthnFeatureEnabled)...)
+
+	return *mapper, allErrs
+}
+
+func validateIssuer(issuer api.Issuer, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	allErrs = append(allErrs, validateURL(issuer.URL, fldPath.Child("url"))...)
+	allErrs = append(allErrs, validateAudiences(issuer.Audiences, fldPath.Child("audiences"))...)
+	allErrs = append(allErrs, validateCertificateAuthority(issuer.CertificateAuthority, fldPath.Child("certificateAuthority"))...)
+
+	return allErrs
+}
+
+func validateURL(issuerURL string, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if len(issuerURL) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath, "URL is required"))
+		return allErrs
+	}
+
+	u, err := url.Parse(issuerURL)
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, issuerURL, err.Error()))
+		return allErrs
+	}
+	if u.Scheme != "https" {
+		allErrs = append(allErrs, field.Invalid(fldPath, issuerURL, "URL scheme must be https"))
+	}
+	if u.User != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, issuerURL, "URL must not contain a username or password"))
+	}
+	if len(u.RawQuery) > 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath, issuerURL, "URL must not contain a query"))
+	}
+	if len(u.Fragment) > 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath, issuerURL, "URL must not contain a fragment"))
+	}
+
+	return allErrs
+}
+
+func validateAudiences(audiences []string, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if len(audiences) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath, fmt.Sprintf(atLeastOneRequiredErrFmt, fldPath)))
+		return allErrs
+	}
+	// This stricter validation is because the --oidc-client-id flag option is singular.
+	// This will be removed when we support multiple audiences with the StructuredAuthenticationConfiguration feature gate.
+	if len(audiences) > 1 {
+		allErrs = append(allErrs, field.TooMany(fldPath, len(audiences), 1))
+		return allErrs
+	}
+
+	for i, audience := range audiences {
+		fldPath := fldPath.Index(i)
+		if len(audience) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath, "audience can't be empty"))
+		}
+	}
+
+	return allErrs
+}
+
+func validateCertificateAuthority(certificateAuthority string, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if len(certificateAuthority) == 0 {
+		return allErrs
+	}
+	_, err := cert.NewPoolFromBytes([]byte(certificateAuthority))
+	if err != nil {
+		allErrs = append(allErrs, field.Invalid(fldPath, "<omitted>", err.Error()))
+	}
+
+	return allErrs
+}
+
+func validateClaimValidationRules(compiler authenticationcel.Compiler, celMapper *authenticationcel.CELMapper, rules []api.ClaimValidationRule, fldPath *field.Path, structuredAuthnFeatureEnabled bool) field.ErrorList {
+	var allErrs field.ErrorList
+
+	seenClaims := sets.NewString()
+	seenExpressions := sets.NewString()
+	var compilationResults []authenticationcel.CompilationResult
+
+	for i, rule := range rules {
+		fldPath := fldPath.Index(i)
+
+		if len(rule.Expression) > 0 && !structuredAuthnFeatureEnabled {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("expression"), rule.Expression, "expression is not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+		}
+
+		switch {
+		case len(rule.Claim) > 0 && len(rule.Expression) > 0:
+			allErrs = append(allErrs, field.Invalid(fldPath, rule.Claim, "claim and expression can't both be set"))
+		case len(rule.Claim) == 0 && len(rule.Expression) == 0:
+			allErrs = append(allErrs, field.Required(fldPath, "claim or expression is required"))
+		case len(rule.Claim) > 0:
+			if len(rule.Message) > 0 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("message"), rule.Message, "message can't be set when claim is set"))
+			}
+			if seenClaims.Has(rule.Claim) {
+				allErrs = append(allErrs, field.Duplicate(fldPath.Child("claim"), rule.Claim))
+			}
+			seenClaims.Insert(rule.Claim)
+		case len(rule.Expression) > 0:
+			if len(rule.RequiredValue) > 0 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("requiredValue"), rule.RequiredValue, "requiredValue can't be set when expression is set"))
+			}
+			if seenExpressions.Has(rule.Expression) {
+				allErrs = append(allErrs, field.Duplicate(fldPath.Child("expression"), rule.Expression))
+				continue
+			}
+			seenExpressions.Insert(rule.Expression)
+
+			compilationResult, err := compileClaimsCELExpression(compiler, &authenticationcel.ClaimValidationCondition{
+				Expression: rule.Expression,
+			}, fldPath.Child("expression"))
+
+			if err != nil {
+				allErrs = append(allErrs, err)
+				continue
+			}
+			if compilationResult != nil {
+				compilationResults = append(compilationResults, *compilationResult)
+			}
+		}
+	}
+
+	if structuredAuthnFeatureEnabled && len(compilationResults) > 0 {
+		celMapper.ClaimValidationRules = authenticationcel.NewClaimsMapper(compilationResults)
+	}
+
+	return allErrs
+}
+
+func validateClaimMappings(compiler authenticationcel.Compiler, celMapper *authenticationcel.CELMapper, m api.ClaimMappings, fldPath *field.Path, structuredAuthnFeatureEnabled bool) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if !structuredAuthnFeatureEnabled {
+		if len(m.Username.Expression) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("username").Child("expression"), m.Username.Expression, "expression is not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+		}
+		if len(m.Groups.Expression) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("groups").Child("expression"), m.Groups.Expression, "expression is not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+		}
+		if len(m.UID.Claim) > 0 || len(m.UID.Expression) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("uid"), "", "uid claim mapping is not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+		}
+		if len(m.Extra) > 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("extra"), "", "extra claim mapping is not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+		}
+	}
+
+	compilationResult, err := validatePrefixClaimOrExpression(compiler, m.Username, fldPath.Child("username"), true, structuredAuthnFeatureEnabled)
+	if err != nil {
+		allErrs = append(allErrs, err...)
+	} else if compilationResult != nil && structuredAuthnFeatureEnabled {
+		celMapper.Username = authenticationcel.NewClaimsMapper([]authenticationcel.CompilationResult{*compilationResult})
+	}
+
+	compilationResult, err = validatePrefixClaimOrExpression(compiler, m.Groups, fldPath.Child("groups"), false, structuredAuthnFeatureEnabled)
+	if err != nil {
+		allErrs = append(allErrs, err...)
+	} else if compilationResult != nil && structuredAuthnFeatureEnabled {
+		celMapper.Groups = authenticationcel.NewClaimsMapper([]authenticationcel.CompilationResult{*compilationResult})
+	}
+
+	switch {
+	case len(m.UID.Claim) > 0 && len(m.UID.Expression) > 0:
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("uid"), "", "claim and expression can't both be set"))
+	case len(m.UID.Expression) > 0:
+		compilationResult, err := compileClaimsCELExpression(compiler, &authenticationcel.ClaimMappingExpression{
+			Expression: m.UID.Expression,
+		}, fldPath.Child("uid").Child("expression"))
+
+		if err != nil {
+			allErrs = append(allErrs, err)
+		} else if structuredAuthnFeatureEnabled && compilationResult != nil {
+			celMapper.UID = authenticationcel.NewClaimsMapper([]authenticationcel.CompilationResult{*compilationResult})
+		}
+	}
+
+	var extraCompilationResults []authenticationcel.CompilationResult
+	seenExtraKeys := sets.NewString()
+
+	for i, mapping := range m.Extra {
+		fldPath := fldPath.Child("extra").Index(i)
+		// Key should be namespaced to the authenticator or authenticator/authorizer pair making use of them.
+		// For instance: "example.org/foo" instead of "foo".
+		// xref: https://github.com/kubernetes/kubernetes/blob/3825e206cb162a7ad7431a5bdf6a065ae8422cf7/staging/src/k8s.io/apiserver/pkg/authentication/user/user.go#L31-L41
+		// IsDomainPrefixedPath checks for non-empty key and that the key is prefixed with a domain name.
+		allErrs = append(allErrs, utilvalidation.IsDomainPrefixedPath(fldPath.Child("key"), mapping.Key)...)
+		if mapping.Key != strings.ToLower(mapping.Key) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("key"), mapping.Key, "key must be lowercase"))
+		}
+		if seenExtraKeys.Has(mapping.Key) {
+			allErrs = append(allErrs, field.Duplicate(fldPath.Child("key"), mapping.Key))
+			continue
+		}
+		seenExtraKeys.Insert(mapping.Key)
+
+		if len(mapping.ValueExpression) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("valueExpression"), "valueExpression is required"))
+			continue
+		}
+
+		compilationResult, err := compileClaimsCELExpression(compiler, &authenticationcel.ExtraMappingExpression{
+			Key:        mapping.Key,
+			Expression: mapping.ValueExpression,
+		}, fldPath.Child("valueExpression"))
+
+		if err != nil {
+			allErrs = append(allErrs, err)
+			continue
+		}
+
+		if compilationResult != nil {
+			extraCompilationResults = append(extraCompilationResults, *compilationResult)
+		}
+	}
+
+	if structuredAuthnFeatureEnabled && len(extraCompilationResults) > 0 {
+		celMapper.Extra = authenticationcel.NewClaimsMapper(extraCompilationResults)
+	}
+
+	return allErrs
+}
+
+func validatePrefixClaimOrExpression(compiler authenticationcel.Compiler, mapping api.PrefixedClaimOrExpression, fldPath *field.Path, claimOrExpressionRequired, structuredAuthnFeatureEnabled bool) (*authenticationcel.CompilationResult, field.ErrorList) {
+	var allErrs field.ErrorList
+
+	var compilationResult *authenticationcel.CompilationResult
+	switch {
+	case len(mapping.Expression) > 0 && len(mapping.Claim) > 0:
+		allErrs = append(allErrs, field.Invalid(fldPath, "", "claim and expression can't both be set"))
+	case len(mapping.Expression) == 0 && len(mapping.Claim) == 0 && claimOrExpressionRequired:
+		allErrs = append(allErrs, field.Required(fldPath, "claim or expression is required"))
+	case len(mapping.Expression) > 0:
+		var err *field.Error
+
+		if mapping.Prefix != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("prefix"), *mapping.Prefix, "prefix can't be set when expression is set"))
+		}
+		compilationResult, err = compileClaimsCELExpression(compiler, &authenticationcel.ClaimMappingExpression{
+			Expression: mapping.Expression,
+		}, fldPath.Child("expression"))
+
+		if err != nil {
+			allErrs = append(allErrs, err)
+		}
+
+	case len(mapping.Claim) > 0:
+		if mapping.Prefix == nil {
+			allErrs = append(allErrs, field.Required(fldPath.Child("prefix"), "prefix is required when claim is set. It can be set to an empty string to disable prefixing"))
+		}
+	}
+
+	return compilationResult, allErrs
+}
+
+func validateUserValidationRules(compiler authenticationcel.Compiler, celMapper *authenticationcel.CELMapper, rules []api.UserValidationRule, fldPath *field.Path, structuredAuthnFeatureEnabled bool) field.ErrorList {
+	var allErrs field.ErrorList
+	var compilationResults []authenticationcel.CompilationResult
+
+	if len(rules) > 0 && !structuredAuthnFeatureEnabled {
+		allErrs = append(allErrs, field.Invalid(fldPath, "", "user validation rules are not supported when StructuredAuthenticationConfiguration feature gate is disabled"))
+	}
+
+	seenExpressions := sets.NewString()
+	for i, rule := range rules {
+		fldPath := fldPath.Index(i)
+
+		if len(rule.Expression) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("expression"), "expression is required"))
+			continue
+		}
+
+		if seenExpressions.Has(rule.Expression) {
+			allErrs = append(allErrs, field.Duplicate(fldPath.Child("expression"), rule.Expression))
+			continue
+		}
+		seenExpressions.Insert(rule.Expression)
+
+		compilationResult, err := compileUserCELExpression(compiler, &authenticationcel.UserValidationCondition{
+			Expression: rule.Expression,
+			Message:    rule.Message,
+		}, fldPath.Child("expression"))
+
+		if err != nil {
+			allErrs = append(allErrs, err)
+			continue
+		}
+
+		if compilationResult != nil {
+			compilationResults = append(compilationResults, *compilationResult)
+		}
+	}
+
+	if structuredAuthnFeatureEnabled && len(compilationResults) > 0 {
+		celMapper.UserValidationRules = authenticationcel.NewUserMapper(compilationResults)
+	}
+
+	return allErrs
+}
+
+func compileClaimsCELExpression(compiler authenticationcel.Compiler, expression authenticationcel.ExpressionAccessor, fldPath *field.Path) (*authenticationcel.CompilationResult, *field.Error) {
+	compilationResult, err := compiler.CompileClaimsExpression(expression)
+	if err != nil {
+		return nil, convertCELErrorToValidationError(fldPath, expression, err)
+	}
+	return &compilationResult, nil
+}
+
+func compileUserCELExpression(compiler authenticationcel.Compiler, expression authenticationcel.ExpressionAccessor, fldPath *field.Path) (*authenticationcel.CompilationResult, *field.Error) {
+	compilationResult, err := compiler.CompileUserExpression(expression)
+	if err != nil {
+		return nil, convertCELErrorToValidationError(fldPath, expression, err)
+	}
+	return &compilationResult, nil
+}
+
+// ValidateAuthorizationConfiguration validates a given AuthorizationConfiguration.
+func ValidateAuthorizationConfiguration(fldPath *field.Path, c *api.AuthorizationConfiguration, knownTypes sets.String, repeatableTypes sets.String) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if len(c.Authorizers) == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("authorizers"), "at least one authorization mode must be defined"))
+	}
+
+	seenAuthorizerTypes := sets.NewString()
+	seenAuthorizerNames := sets.NewString()
+	for i, a := range c.Authorizers {
+		fldPath := fldPath.Child("authorizers").Index(i)
+		aType := string(a.Type)
+		if aType == "" {
+			allErrs = append(allErrs, field.Required(fldPath.Child("type"), ""))
+			continue
+		}
+		if !knownTypes.Has(aType) {
+			allErrs = append(allErrs, field.NotSupported(fldPath.Child("type"), aType, knownTypes.List()))
+			continue
+		}
+		if seenAuthorizerTypes.Has(aType) && !repeatableTypes.Has(aType) {
+			allErrs = append(allErrs, field.Duplicate(fldPath.Child("type"), aType))
+			continue
+		}
+		seenAuthorizerTypes.Insert(aType)
+
+		if len(a.Name) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("name"), ""))
+		} else if seenAuthorizerNames.Has(a.Name) {
+			allErrs = append(allErrs, field.Duplicate(fldPath.Child("name"), a.Name))
+		} else if errs := utilvalidation.IsDNS1123Subdomain(a.Name); len(errs) != 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("name"), a.Name, fmt.Sprintf("authorizer name is invalid: %s", strings.Join(errs, ", "))))
+		}
+		seenAuthorizerNames.Insert(a.Name)
+
+		switch a.Type {
+		case api.TypeWebhook:
+			if a.Webhook == nil {
+				allErrs = append(allErrs, field.Required(fldPath.Child("webhook"), "required when type=Webhook"))
+				continue
+			}
+			allErrs = append(allErrs, ValidateWebhookConfiguration(fldPath, a.Webhook)...)
+		default:
+			if a.Webhook != nil {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("webhook"), "non-null", "may only be specified when type=Webhook"))
+			}
+		}
+	}
+
+	return allErrs
+}
+
+func ValidateWebhookConfiguration(fldPath *field.Path, c *api.WebhookConfiguration) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if c.Timeout.Duration == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("timeout"), ""))
+	} else if c.Timeout.Duration > 30*time.Second || c.Timeout.Duration < 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("timeout"), c.Timeout.Duration.String(), "must be > 0s and <= 30s"))
+	}
+
+	if c.AuthorizedTTL.Duration == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("authorizedTTL"), ""))
+	} else if c.AuthorizedTTL.Duration < 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("authorizedTTL"), c.AuthorizedTTL.Duration.String(), "must be > 0s"))
+	}
+
+	if c.UnauthorizedTTL.Duration == 0 {
+		allErrs = append(allErrs, field.Required(fldPath.Child("unauthorizedTTL"), ""))
+	} else if c.UnauthorizedTTL.Duration < 0 {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("unauthorizedTTL"), c.UnauthorizedTTL.Duration.String(), "must be > 0s"))
+	}
+
+	switch c.SubjectAccessReviewVersion {
+	case "":
+		allErrs = append(allErrs, field.Required(fldPath.Child("subjectAccessReviewVersion"), ""))
+	case "v1":
+		_ = &v1.SubjectAccessReview{}
+	case "v1beta1":
+		_ = &v1beta1.SubjectAccessReview{}
+	default:
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("subjectAccessReviewVersion"), c.SubjectAccessReviewVersion, []string{"v1", "v1beta1"}))
+	}
+
+	switch c.MatchConditionSubjectAccessReviewVersion {
+	case "":
+		if len(c.MatchConditions) > 0 {
+			allErrs = append(allErrs, field.Required(fldPath.Child("matchConditionSubjectAccessReviewVersion"), "required if match conditions are specified"))
+		}
+	case "v1":
+		_ = &v1.SubjectAccessReview{}
+	default:
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("matchConditionSubjectAccessReviewVersion"), c.MatchConditionSubjectAccessReviewVersion, []string{"v1"}))
+	}
+
+	switch c.FailurePolicy {
+	case "":
+		allErrs = append(allErrs, field.Required(fldPath.Child("failurePolicy"), ""))
+	case api.FailurePolicyNoOpinion, api.FailurePolicyDeny:
+	default:
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("failurePolicy"), c.FailurePolicy, []string{"NoOpinion", "Deny"}))
+	}
+
+	switch c.ConnectionInfo.Type {
+	case "":
+		allErrs = append(allErrs, field.Required(fldPath.Child("connectionInfo", "type"), ""))
+	case api.AuthorizationWebhookConnectionInfoTypeInCluster:
+		if c.ConnectionInfo.KubeConfigFile != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("connectionInfo", "kubeConfigFile"), *c.ConnectionInfo.KubeConfigFile, "can only be set when type=KubeConfigFile"))
+		}
+	case api.AuthorizationWebhookConnectionInfoTypeKubeConfigFile:
+		if c.ConnectionInfo.KubeConfigFile == nil || *c.ConnectionInfo.KubeConfigFile == "" {
+			allErrs = append(allErrs, field.Required(fldPath.Child("connectionInfo", "kubeConfigFile"), ""))
+		} else if !filepath.IsAbs(*c.ConnectionInfo.KubeConfigFile) {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("connectionInfo", "kubeConfigFile"), *c.ConnectionInfo.KubeConfigFile, "must be an absolute path"))
+		} else if info, err := os.Stat(*c.ConnectionInfo.KubeConfigFile); err != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("connectionInfo", "kubeConfigFile"), *c.ConnectionInfo.KubeConfigFile, fmt.Sprintf("error loading file: %v", err)))
+		} else if !info.Mode().IsRegular() {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("connectionInfo", "kubeConfigFile"), *c.ConnectionInfo.KubeConfigFile, "must be a regular file"))
+		}
+	default:
+		allErrs = append(allErrs, field.NotSupported(fldPath.Child("connectionInfo", "type"), c.ConnectionInfo, []string{api.AuthorizationWebhookConnectionInfoTypeInCluster, api.AuthorizationWebhookConnectionInfoTypeKubeConfigFile}))
+	}
+
+	_, errs := compileMatchConditions(c.MatchConditions, fldPath, utilfeature.DefaultFeatureGate.Enabled(features.StructuredAuthorizationConfiguration))
+	allErrs = append(allErrs, errs...)
+
+	return allErrs
+}
+
+// ValidateAndCompileMatchConditions validates a given webhook's matchConditions.
+// This is exported for use in authz package.
+func ValidateAndCompileMatchConditions(matchConditions []api.WebhookMatchCondition) (*authorizationcel.CELMatcher, field.ErrorList) {
+	return compileMatchConditions(matchConditions, nil, utilfeature.DefaultFeatureGate.Enabled(features.StructuredAuthorizationConfiguration))
+}
+
+func compileMatchConditions(matchConditions []api.WebhookMatchCondition, fldPath *field.Path, structuredAuthzFeatureEnabled bool) (*authorizationcel.CELMatcher, field.ErrorList) {
+	var allErrs field.ErrorList
+	// should fail when match conditions are used without feature enabled
+	if len(matchConditions) > 0 && !structuredAuthzFeatureEnabled {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("matchConditions"), "", "matchConditions are not supported when StructuredAuthorizationConfiguration feature gate is disabled"))
+	}
+	if len(matchConditions) > 64 {
+		allErrs = append(allErrs, field.TooMany(fldPath.Child("matchConditions"), len(matchConditions), 64))
+		return nil, allErrs
+	}
+
+	compiler := authorizationcel.NewCompiler(environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion()))
+	seenExpressions := sets.NewString()
+	var compilationResults []authorizationcel.CompilationResult
+
+	for i, condition := range matchConditions {
+		fldPath := fldPath.Child("matchConditions").Index(i).Child("expression")
+		if len(strings.TrimSpace(condition.Expression)) == 0 {
+			allErrs = append(allErrs, field.Required(fldPath, ""))
+			continue
+		}
+		if seenExpressions.Has(condition.Expression) {
+			allErrs = append(allErrs, field.Duplicate(fldPath, condition.Expression))
+			continue
+		}
+		seenExpressions.Insert(condition.Expression)
+		compilationResult, err := compileMatchConditionsExpression(fldPath, compiler, condition.Expression)
+		if err != nil {
+			allErrs = append(allErrs, err)
+			continue
+		}
+		compilationResults = append(compilationResults, compilationResult)
+	}
+	if len(compilationResults) == 0 {
+		return nil, allErrs
+	}
+	return &authorizationcel.CELMatcher{
+		CompilationResults: compilationResults,
+	}, allErrs
+}
+
+func compileMatchConditionsExpression(fldPath *field.Path, compiler authorizationcel.Compiler, expression string) (authorizationcel.CompilationResult, *field.Error) {
+	authzExpression := &authorizationcel.SubjectAccessReviewMatchCondition{
+		Expression: expression,
+	}
+	compilationResult, err := compiler.CompileCELExpression(authzExpression)
+	if err != nil {
+		return compilationResult, convertCELErrorToValidationError(fldPath, authzExpression, err)
+	}
+	return compilationResult, nil
+}
+
+func convertCELErrorToValidationError(fldPath *field.Path, expression authorizationcel.ExpressionAccessor, err error) *field.Error {
+	var celErr *cel.Error
+	if errors.As(err, &celErr) {
+		switch celErr.Type {
+		case cel.ErrorTypeRequired:
+			return field.Required(fldPath, celErr.Detail)
+		case cel.ErrorTypeInvalid:
+			return field.Invalid(fldPath, expression.GetExpression(), celErr.Detail)
+		default:
+			return field.InternalError(fldPath, celErr)
+		}
+	}
+	return field.InternalError(fldPath, fmt.Errorf("error is not cel error: %w", err))
+}

vendor/k8s.io/apiserver/pkg/apis/apiserver/zz_generated.deepcopy.go

@@ -78,6 +78,147 @@ func (in *AdmissionPluginConfiguration) DeepCopy() *AdmissionPluginConfiguration
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthenticationConfiguration) DeepCopyInto(out *AuthenticationConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.JWT != nil {
+		in, out := &in.JWT, &out.JWT
+		*out = make([]JWTAuthenticator, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthenticationConfiguration.
+func (in *AuthenticationConfiguration) DeepCopy() *AuthenticationConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthenticationConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AuthenticationConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizationConfiguration) DeepCopyInto(out *AuthorizationConfiguration) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	if in.Authorizers != nil {
+		in, out := &in.Authorizers, &out.Authorizers
+		*out = make([]AuthorizerConfiguration, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizationConfiguration.
+func (in *AuthorizationConfiguration) DeepCopy() *AuthorizationConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizationConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *AuthorizationConfiguration) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AuthorizerConfiguration) DeepCopyInto(out *AuthorizerConfiguration) {
+	*out = *in
+	if in.Webhook != nil {
+		in, out := &in.Webhook, &out.Webhook
+		*out = new(WebhookConfiguration)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorizerConfiguration.
+func (in *AuthorizerConfiguration) DeepCopy() *AuthorizerConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(AuthorizerConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimMappings) DeepCopyInto(out *ClaimMappings) {
+	*out = *in
+	in.Username.DeepCopyInto(&out.Username)
+	in.Groups.DeepCopyInto(&out.Groups)
+	out.UID = in.UID
+	if in.Extra != nil {
+		in, out := &in.Extra, &out.Extra
+		*out = make([]ExtraMapping, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimMappings.
+func (in *ClaimMappings) DeepCopy() *ClaimMappings {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimMappings)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimOrExpression) DeepCopyInto(out *ClaimOrExpression) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimOrExpression.
+func (in *ClaimOrExpression) DeepCopy() *ClaimOrExpression {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimOrExpression)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ClaimValidationRule) DeepCopyInto(out *ClaimValidationRule) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClaimValidationRule.
+func (in *ClaimValidationRule) DeepCopy() *ClaimValidationRule {
+	if in == nil {
+		return nil
+	}
+	out := new(ClaimValidationRule)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Connection) DeepCopyInto(out *Connection) {
 	*out = *in
@@ -148,6 +289,92 @@ func (in *EgressSelectorConfiguration) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ExtraMapping) DeepCopyInto(out *ExtraMapping) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraMapping.
+func (in *ExtraMapping) DeepCopy() *ExtraMapping {
+	if in == nil {
+		return nil
+	}
+	out := new(ExtraMapping)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Issuer) DeepCopyInto(out *Issuer) {
+	*out = *in
+	if in.Audiences != nil {
+		in, out := &in.Audiences, &out.Audiences
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Issuer.
+func (in *Issuer) DeepCopy() *Issuer {
+	if in == nil {
+		return nil
+	}
+	out := new(Issuer)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *JWTAuthenticator) DeepCopyInto(out *JWTAuthenticator) {
+	*out = *in
+	in.Issuer.DeepCopyInto(&out.Issuer)
+	if in.ClaimValidationRules != nil {
+		in, out := &in.ClaimValidationRules, &out.ClaimValidationRules
+		*out = make([]ClaimValidationRule, len(*in))
+		copy(*out, *in)
+	}
+	in.ClaimMappings.DeepCopyInto(&out.ClaimMappings)
+	if in.UserValidationRules != nil {
+		in, out := &in.UserValidationRules, &out.UserValidationRules
+		*out = make([]UserValidationRule, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JWTAuthenticator.
+func (in *JWTAuthenticator) DeepCopy() *JWTAuthenticator {
+	if in == nil {
+		return nil
+	}
+	out := new(JWTAuthenticator)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PrefixedClaimOrExpression) DeepCopyInto(out *PrefixedClaimOrExpression) {
+	*out = *in
+	if in.Prefix != nil {
+		in, out := &in.Prefix, &out.Prefix
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrefixedClaimOrExpression.
+func (in *PrefixedClaimOrExpression) DeepCopy() *PrefixedClaimOrExpression {
+	if in == nil {
+		return nil
+	}
+	out := new(PrefixedClaimOrExpression)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TCPTransport) DeepCopyInto(out *TCPTransport) {
 	*out = *in
@@ -252,3 +479,81 @@ func (in *UDSTransport) DeepCopy() *UDSTransport {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UserValidationRule) DeepCopyInto(out *UserValidationRule) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserValidationRule.
+func (in *UserValidationRule) DeepCopy() *UserValidationRule {
+	if in == nil {
+		return nil
+	}
+	out := new(UserValidationRule)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookConfiguration) DeepCopyInto(out *WebhookConfiguration) {
+	*out = *in
+	out.AuthorizedTTL = in.AuthorizedTTL
+	out.UnauthorizedTTL = in.UnauthorizedTTL
+	out.Timeout = in.Timeout
+	in.ConnectionInfo.DeepCopyInto(&out.ConnectionInfo)
+	if in.MatchConditions != nil {
+		in, out := &in.MatchConditions, &out.MatchConditions
+		*out = make([]WebhookMatchCondition, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConfiguration.
+func (in *WebhookConfiguration) DeepCopy() *WebhookConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookConnectionInfo) DeepCopyInto(out *WebhookConnectionInfo) {
+	*out = *in
+	if in.KubeConfigFile != nil {
+		in, out := &in.KubeConfigFile, &out.KubeConfigFile
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookConnectionInfo.
+func (in *WebhookConnectionInfo) DeepCopy() *WebhookConnectionInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookConnectionInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *WebhookMatchCondition) DeepCopyInto(out *WebhookMatchCondition) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WebhookMatchCondition.
+func (in *WebhookMatchCondition) DeepCopy() *WebhookMatchCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(WebhookMatchCondition)
+	in.DeepCopyInto(out)
+	return out
+}

vendor/k8s.io/apiserver/pkg/apis/audit/types.go

@@ -235,10 +235,10 @@ type PolicyRule struct {
 	Namespaces []string
 
 	// NonResourceURLs is a set of URL paths that should be audited.
-	// *s are allowed, but only as the full, final step in the path.
+	// `*`s are allowed, but only as the full, final step in the path.
 	// Examples:
-	//  "/metrics" - Log requests for apiserver metrics
-	//  "/healthz*" - Log all health checks
+	//  `/metrics` - Log requests for apiserver metrics
+	//  `/healthz*` - Log all health checks
 	// +optional
 	NonResourceURLs []string
 
@@ -269,11 +269,11 @@ type GroupResources struct {
 	// Resources is a list of resources this rule applies to.
 	//
 	// For example:
-	// 'pods' matches pods.
-	// 'pods/log' matches the log subresource of pods.
-	// '*' matches all resources and their subresources.
-	// 'pods/*' matches all subresources of pods.
-	// '*/scale' matches all scale subresources.
+	// - `pods` matches pods.
+	// - `pods/log` matches the log subresource of pods.
+	// - `*` matches all resources and their subresources.
+	// - `pods/*` matches all subresources of pods.
+	// - `*/scale` matches all scale subresources.
 	//
 	// If wildcard is present, the validation rule will ensure resources do not
 	// overlap with each other.

vendor/k8s.io/apiserver/pkg/apis/audit/v1/generated.proto

@@ -129,11 +129,11 @@ message GroupResources {
   // Resources is a list of resources this rule applies to.
   //
   // For example:
-  // 'pods' matches pods.
-  // 'pods/log' matches the log subresource of pods.
-  // '*' matches all resources and their subresources.
-  // 'pods/*' matches all subresources of pods.
-  // '*/scale' matches all scale subresources.
+  // - `pods` matches pods.
+  // - `pods/log` matches the log subresource of pods.
+  // - `*` matches all resources and their subresources.
+  // - `pods/*` matches all subresources of pods.
+  // - `*/scale` matches all scale subresources.
   //
   // If wildcard is present, the validation rule will ensure resources do not
   // overlap with each other.
@@ -248,10 +248,10 @@ message PolicyRule {
   repeated string namespaces = 6;
 
   // NonResourceURLs is a set of URL paths that should be audited.
-  // *s are allowed, but only as the full, final step in the path.
+  // `*`s are allowed, but only as the full, final step in the path.
   // Examples:
-  //  "/metrics" - Log requests for apiserver metrics
-  //  "/healthz*" - Log all health checks
+  // - `/metrics` - Log requests for apiserver metrics
+  // - `/healthz*` - Log all health checks
   // +optional
   repeated string nonResourceURLs = 7;
 

vendor/k8s.io/apiserver/pkg/apis/audit/v1/types.go

@@ -229,10 +229,10 @@ type PolicyRule struct {
 	Namespaces []string `json:"namespaces,omitempty" protobuf:"bytes,6,rep,name=namespaces"`
 
 	// NonResourceURLs is a set of URL paths that should be audited.
-	// *s are allowed, but only as the full, final step in the path.
+	// `*`s are allowed, but only as the full, final step in the path.
 	// Examples:
-	//  "/metrics" - Log requests for apiserver metrics
-	//  "/healthz*" - Log all health checks
+	// - `/metrics` - Log requests for apiserver metrics
+	// - `/healthz*` - Log all health checks
 	// +optional
 	NonResourceURLs []string `json:"nonResourceURLs,omitempty" protobuf:"bytes,7,rep,name=nonResourceURLs"`
 
@@ -263,11 +263,11 @@ type GroupResources struct {
 	// Resources is a list of resources this rule applies to.
 	//
 	// For example:
-	// 'pods' matches pods.
-	// 'pods/log' matches the log subresource of pods.
-	// '*' matches all resources and their subresources.
-	// 'pods/*' matches all subresources of pods.
-	// '*/scale' matches all scale subresources.
+	// - `pods` matches pods.
+	// - `pods/log` matches the log subresource of pods.
+	// - `*` matches all resources and their subresources.
+	// - `pods/*` matches all subresources of pods.
+	// - `*/scale` matches all scale subresources.
 	//
 	// If wildcard is present, the validation rule will ensure resources do not
 	// overlap with each other.

vendor/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go

@@ -19,11 +19,11 @@ package bootstrap
 import (
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
-	flowcontrol "k8s.io/api/flowcontrol/v1beta3"
+	flowcontrol "k8s.io/api/flowcontrol/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 )
 
 // The objects that define an apiserver's initial behavior.  The
@@ -90,8 +90,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementExempt,
 			Exempt: &flowcontrol.ExemptPriorityLevelConfiguration{
-				NominalConcurrencyShares: pointer.Int32(0),
-				LendablePercent:          pointer.Int32(0),
+				NominalConcurrencyShares: ptr.To(int32(0)),
+				LendablePercent:          ptr.To(int32(0)),
 			},
 		},
 	)
@@ -100,8 +100,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 5,
-				LendablePercent:          pointer.Int32(0),
+				NominalConcurrencyShares: ptr.To(int32(5)),
+				LendablePercent:          ptr.To(int32(0)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeReject,
 				},
@@ -173,8 +173,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 30,
-				LendablePercent:          pointer.Int32(33),
+				NominalConcurrencyShares: ptr.To(int32(30)),
+				LendablePercent:          ptr.To(int32(33)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -190,8 +190,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 40,
-				LendablePercent:          pointer.Int32(25),
+				NominalConcurrencyShares: ptr.To(int32(40)),
+				LendablePercent:          ptr.To(int32(25)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -208,8 +208,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 10,
-				LendablePercent:          pointer.Int32(0),
+				NominalConcurrencyShares: ptr.To(int32(10)),
+				LendablePercent:          ptr.To(int32(0)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -226,8 +226,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 40,
-				LendablePercent:          pointer.Int32(50),
+				NominalConcurrencyShares: ptr.To(int32(40)),
+				LendablePercent:          ptr.To(int32(50)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -244,8 +244,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 100,
-				LendablePercent:          pointer.Int32(90),
+				NominalConcurrencyShares: ptr.To(int32(100)),
+				LendablePercent:          ptr.To(int32(90)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -262,8 +262,8 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				NominalConcurrencyShares: 20,
-				LendablePercent:          pointer.Int32(50),
+				NominalConcurrencyShares: ptr.To(int32(20)),
+				LendablePercent:          ptr.To(int32(50)),
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{

vendor/k8s.io/apiserver/pkg/authentication/cel/compile.go

@@ -0,0 +1,154 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"fmt"
+
+	"github.com/google/cel-go/cel"
+
+	"k8s.io/apimachinery/pkg/util/version"
+	apiservercel "k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
+)
+
+const (
+	claimsVarName = "claims"
+	userVarName   = "user"
+)
+
+// compiler implements the Compiler interface.
+type compiler struct {
+	// varEnvs is a map of CEL environments, keyed by the name of the CEL variable.
+	// The CEL variable is available to the expression.
+	// We have 2 environments, one for claims and one for user.
+	varEnvs map[string]*environment.EnvSet
+}
+
+// NewCompiler returns a new Compiler.
+func NewCompiler(env *environment.EnvSet) Compiler {
+	return &compiler{
+		varEnvs: mustBuildEnvs(env),
+	}
+}
+
+// CompileClaimsExpression compiles the given expressionAccessor into a CEL program that can be evaluated.
+// The claims CEL variable is available to the expression.
+func (c compiler) CompileClaimsExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error) {
+	return c.compile(expressionAccessor, claimsVarName)
+}
+
+// CompileUserExpression compiles the given expressionAccessor into a CEL program that can be evaluated.
+// The user CEL variable is available to the expression.
+func (c compiler) CompileUserExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error) {
+	return c.compile(expressionAccessor, userVarName)
+}
+
+func (c compiler) compile(expressionAccessor ExpressionAccessor, envVarName string) (CompilationResult, error) {
+	resultError := func(errorString string, errType apiservercel.ErrorType) (CompilationResult, error) {
+		return CompilationResult{}, &apiservercel.Error{
+			Type:   errType,
+			Detail: errorString,
+		}
+	}
+
+	env, err := c.varEnvs[envVarName].Env(environment.StoredExpressions)
+	if err != nil {
+		return resultError(fmt.Sprintf("unexpected error loading CEL environment: %v", err), apiservercel.ErrorTypeInternal)
+	}
+
+	ast, issues := env.Compile(expressionAccessor.GetExpression())
+	if issues != nil {
+		return resultError("compilation failed: "+issues.String(), apiservercel.ErrorTypeInvalid)
+	}
+
+	found := false
+	returnTypes := expressionAccessor.ReturnTypes()
+	for _, returnType := range returnTypes {
+		if ast.OutputType() == returnType || cel.AnyType == returnType {
+			found = true
+			break
+		}
+	}
+	if !found {
+		var reason string
+		if len(returnTypes) == 1 {
+			reason = fmt.Sprintf("must evaluate to %v", returnTypes[0].String())
+		} else {
+			reason = fmt.Sprintf("must evaluate to one of %v", returnTypes)
+		}
+
+		return resultError(reason, apiservercel.ErrorTypeInvalid)
+	}
+
+	if _, err = cel.AstToCheckedExpr(ast); err != nil {
+		// should be impossible since env.Compile returned no issues
+		return resultError("unexpected compilation error: "+err.Error(), apiservercel.ErrorTypeInternal)
+	}
+	prog, err := env.Program(ast)
+	if err != nil {
+		return resultError("program instantiation failed: "+err.Error(), apiservercel.ErrorTypeInternal)
+	}
+
+	return CompilationResult{
+		Program:            prog,
+		ExpressionAccessor: expressionAccessor,
+	}, nil
+}
+
+func buildUserType() *apiservercel.DeclType {
+	field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
+		return apiservercel.NewDeclField(name, declType, required, nil, nil)
+	}
+	fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
+		result := make(map[string]*apiservercel.DeclField, len(fields))
+		for _, f := range fields {
+			result[f.Name] = f
+		}
+		return result
+	}
+
+	return apiservercel.NewObjectType("kubernetes.UserInfo", fields(
+		field("username", apiservercel.StringType, false),
+		field("uid", apiservercel.StringType, false),
+		field("groups", apiservercel.NewListType(apiservercel.StringType, -1), false),
+		field("extra", apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewListType(apiservercel.StringType, -1), -1), false),
+	))
+}
+
+func mustBuildEnvs(baseEnv *environment.EnvSet) map[string]*environment.EnvSet {
+	buildEnvSet := func(envOpts []cel.EnvOption, declTypes []*apiservercel.DeclType) *environment.EnvSet {
+		env, err := baseEnv.Extend(environment.VersionedOptions{
+			IntroducedVersion: version.MajorMinor(1, 0),
+			EnvOptions:        envOpts,
+			DeclTypes:         declTypes,
+		})
+		if err != nil {
+			panic(fmt.Sprintf("environment misconfigured: %v", err))
+		}
+		return env
+	}
+
+	userType := buildUserType()
+	claimsType := apiservercel.NewMapType(apiservercel.StringType, apiservercel.AnyType, -1)
+
+	envs := make(map[string]*environment.EnvSet, 2) // build two environments, one for claims and one for user
+	envs[claimsVarName] = buildEnvSet([]cel.EnvOption{cel.Variable(claimsVarName, claimsType.CelType())}, []*apiservercel.DeclType{claimsType})
+	envs[userVarName] = buildEnvSet([]cel.EnvOption{cel.Variable(userVarName, userType.CelType())}, []*apiservercel.DeclType{userType})
+
+	return envs
+}

vendor/k8s.io/apiserver/pkg/authentication/cel/interface.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cel contains the CEL related interfaces and structs for authentication.
+package cel
+
+import (
+	"context"
+
+	celgo "github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types/ref"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+// ExpressionAccessor is an interface that provides access to a CEL expression.
+type ExpressionAccessor interface {
+	GetExpression() string
+	ReturnTypes() []*celgo.Type
+}
+
+// CompilationResult represents a compiled validations expression.
+type CompilationResult struct {
+	Program            celgo.Program
+	ExpressionAccessor ExpressionAccessor
+}
+
+// EvaluationResult contains the minimal required fields and metadata of a cel evaluation
+type EvaluationResult struct {
+	EvalResult         ref.Val
+	ExpressionAccessor ExpressionAccessor
+}
+
+// Compiler provides a CEL expression compiler configured with the desired authentication related CEL variables.
+type Compiler interface {
+	CompileClaimsExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error)
+	CompileUserExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error)
+}
+
+// ClaimsMapper provides a CEL expression mapper configured with the claims CEL variable.
+type ClaimsMapper interface {
+	// EvalClaimMapping evaluates the given claim mapping expression and returns a EvaluationResult.
+	// This is used for username, groups and uid claim mapping that contains a single expression.
+	EvalClaimMapping(ctx context.Context, claims *unstructured.Unstructured) (EvaluationResult, error)
+	// EvalClaimMappings evaluates the given expressions and returns a list of EvaluationResult.
+	// This is used for extra claim mapping and claim validation that contains a list of expressions.
+	EvalClaimMappings(ctx context.Context, claims *unstructured.Unstructured) ([]EvaluationResult, error)
+}
+
+// UserMapper provides a CEL expression mapper configured with the user CEL variable.
+type UserMapper interface {
+	// EvalUser evaluates the given user expressions and returns a list of EvaluationResult.
+	// This is used for user validation that contains a list of expressions.
+	EvalUser(ctx context.Context, userInfo *unstructured.Unstructured) ([]EvaluationResult, error)
+}
+
+var _ ExpressionAccessor = &ClaimMappingExpression{}
+
+// ClaimMappingExpression is a CEL expression that maps a claim.
+type ClaimMappingExpression struct {
+	Expression string
+}
+
+// GetExpression returns the CEL expression.
+func (v *ClaimMappingExpression) GetExpression() string {
+	return v.Expression
+}
+
+// ReturnTypes returns the CEL expression return types.
+func (v *ClaimMappingExpression) ReturnTypes() []*celgo.Type {
+	// return types is only used for validation. The claims variable that's available
+	// to the claim mapping expressions is a map[string]interface{}, so we can't
+	// really know what the return type is during compilation. Strict type checking
+	// is done during evaluation.
+	return []*celgo.Type{celgo.AnyType}
+}
+
+var _ ExpressionAccessor = &ClaimValidationCondition{}
+
+// ClaimValidationCondition is a CEL expression that validates a claim.
+type ClaimValidationCondition struct {
+	Expression string
+	Message    string
+}
+
+// GetExpression returns the CEL expression.
+func (v *ClaimValidationCondition) GetExpression() string {
+	return v.Expression
+}
+
+// ReturnTypes returns the CEL expression return types.
+func (v *ClaimValidationCondition) ReturnTypes() []*celgo.Type {
+	return []*celgo.Type{celgo.BoolType}
+}
+
+var _ ExpressionAccessor = &ExtraMappingExpression{}
+
+// ExtraMappingExpression is a CEL expression that maps an extra to a list of values.
+type ExtraMappingExpression struct {
+	Key        string
+	Expression string
+}
+
+// GetExpression returns the CEL expression.
+func (v *ExtraMappingExpression) GetExpression() string {
+	return v.Expression
+}
+
+// ReturnTypes returns the CEL expression return types.
+func (v *ExtraMappingExpression) ReturnTypes() []*celgo.Type {
+	// return types is only used for validation. The claims variable that's available
+	// to the claim mapping expressions is a map[string]interface{}, so we can't
+	// really know what the return type is during compilation. Strict type checking
+	// is done during evaluation.
+	return []*celgo.Type{celgo.AnyType}
+}
+
+var _ ExpressionAccessor = &UserValidationCondition{}
+
+// UserValidationCondition is a CEL expression that validates a User.
+type UserValidationCondition struct {
+	Expression string
+	Message    string
+}
+
+// GetExpression returns the CEL expression.
+func (v *UserValidationCondition) GetExpression() string {
+	return v.Expression
+}
+
+// ReturnTypes returns the CEL expression return types.
+func (v *UserValidationCondition) ReturnTypes() []*celgo.Type {
+	return []*celgo.Type{celgo.BoolType}
+}

vendor/k8s.io/apiserver/pkg/authentication/cel/mapper.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+var _ ClaimsMapper = &mapper{}
+var _ UserMapper = &mapper{}
+
+// mapper implements the ClaimsMapper and UserMapper interface.
+type mapper struct {
+	compilationResults []CompilationResult
+}
+
+// CELMapper is a struct that holds the compiled expressions for
+// username, groups, uid, extra, claimValidation and userValidation
+type CELMapper struct {
+	Username             ClaimsMapper
+	Groups               ClaimsMapper
+	UID                  ClaimsMapper
+	Extra                ClaimsMapper
+	ClaimValidationRules ClaimsMapper
+	UserValidationRules  UserMapper
+}
+
+// NewClaimsMapper returns a new ClaimsMapper.
+func NewClaimsMapper(compilationResults []CompilationResult) ClaimsMapper {
+	return &mapper{
+		compilationResults: compilationResults,
+	}
+}
+
+// NewUserMapper returns a new UserMapper.
+func NewUserMapper(compilationResults []CompilationResult) UserMapper {
+	return &mapper{
+		compilationResults: compilationResults,
+	}
+}
+
+// EvalClaimMapping evaluates the given claim mapping expression and returns a EvaluationResult.
+func (m *mapper) EvalClaimMapping(ctx context.Context, claims *unstructured.Unstructured) (EvaluationResult, error) {
+	results, err := m.eval(ctx, map[string]interface{}{claimsVarName: claims.Object})
+	if err != nil {
+		return EvaluationResult{}, err
+	}
+	if len(results) != 1 {
+		return EvaluationResult{}, fmt.Errorf("expected 1 evaluation result, got %d", len(results))
+	}
+	return results[0], nil
+}
+
+// EvalClaimMappings evaluates the given expressions and returns a list of EvaluationResult.
+func (m *mapper) EvalClaimMappings(ctx context.Context, claims *unstructured.Unstructured) ([]EvaluationResult, error) {
+	return m.eval(ctx, map[string]interface{}{claimsVarName: claims.Object})
+}
+
+// EvalUser evaluates the given user expressions and returns a list of EvaluationResult.
+func (m *mapper) EvalUser(ctx context.Context, userInfo *unstructured.Unstructured) ([]EvaluationResult, error) {
+	return m.eval(ctx, map[string]interface{}{userVarName: userInfo.Object})
+}
+
+func (m *mapper) eval(ctx context.Context, input map[string]interface{}) ([]EvaluationResult, error) {
+	evaluations := make([]EvaluationResult, len(m.compilationResults))
+
+	for i, compilationResult := range m.compilationResults {
+		var evaluation = &evaluations[i]
+		evaluation.ExpressionAccessor = compilationResult.ExpressionAccessor
+
+		evalResult, _, err := compilationResult.Program.ContextEval(ctx, input)
+		if err != nil {
+			return nil, fmt.Errorf("expression '%s' resulted in error: %w", compilationResult.ExpressionAccessor.GetExpression(), err)
+		}
+
+		evaluation.EvalResult = evalResult
+	}
+
+	return evaluations, nil
+}

vendor/k8s.io/apiserver/pkg/authentication/request/x509/x509.go

@@ -148,6 +148,33 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.R
 		}
 	}
 
+	/*
+			kubernetes mutual (2-way) x509 between client and apiserver:
+
+				1. apiserver sending its apiserver certificate along with its publickey to client
+				2. client verifies the apiserver certificate sent against its cluster certificate authority data
+				3. client sending its client certificate along with its public key to the apiserver
+				>4. apiserver verifies the client certificate sent against its cluster certificate authority data
+
+		    	description:
+					here, with this function,
+					client certificate and pub key sent during the handshake process
+					are verified by apiserver against its cluster certificate authority data
+
+				normal args related to this stage:
+					--client-ca-file string   If set, any request presenting a client certificate signed by
+						one of the authorities in the client-ca-file is authenticated with an identity
+						corresponding to the CommonName of the client certificate.
+
+					(retrievable from "kube-apiserver --help" command)
+					(suggested by @deads2k)
+
+				see also:
+					- for the step 1, see: staging/src/k8s.io/apiserver/pkg/server/options/serving.go
+					- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
+					- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
+	*/
+
 	remaining := req.TLS.PeerCertificates[0].NotAfter.Sub(time.Now())
 	clientCertificateExpirationHistogram.WithContext(req.Context()).Observe(remaining.Seconds())
 	chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)

vendor/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go

@@ -36,12 +36,21 @@ const (
 	ServiceAccountUsernameSeparator = ":"
 	ServiceAccountGroupPrefix       = "system:serviceaccounts:"
 	AllServiceAccountsGroup         = "system:serviceaccounts"
+	// CredentialIDKey is the key used in a user's "extra" to specify the unique
+	// identifier for this identity document).
+	CredentialIDKey = "authentication.kubernetes.io/credential-id"
 	// PodNameKey is the key used in a user's "extra" to specify the pod name of
 	// the authenticating request.
 	PodNameKey = "authentication.kubernetes.io/pod-name"
 	// PodUIDKey is the key used in a user's "extra" to specify the pod UID of
 	// the authenticating request.
 	PodUIDKey = "authentication.kubernetes.io/pod-uid"
+	// NodeNameKey is the key used in a user's "extra" to specify the node name of
+	// the authenticating request.
+	NodeNameKey = "authentication.kubernetes.io/node-name"
+	// NodeUIDKey is the key used in a user's "extra" to specify the node UID of
+	// the authenticating request.
+	NodeUIDKey = "authentication.kubernetes.io/node-uid"
 )
 
 // MakeUsername generates a username from the given namespace and ServiceAccount name.
@@ -119,6 +128,8 @@ func UserInfo(namespace, name, uid string) user.Info {
 type ServiceAccountInfo struct {
 	Name, Namespace, UID string
 	PodName, PodUID      string
+	CredentialID         string
+	NodeName, NodeUID    string
 }
 
 func (sa *ServiceAccountInfo) UserInfo() user.Info {
@@ -127,15 +138,43 @@ func (sa *ServiceAccountInfo) UserInfo() user.Info {
 		UID:    sa.UID,
 		Groups: MakeGroupNames(sa.Namespace),
 	}
+
 	if sa.PodName != "" && sa.PodUID != "" {
-		info.Extra = map[string][]string{
-			PodNameKey: {sa.PodName},
-			PodUIDKey:  {sa.PodUID},
+		if info.Extra == nil {
+			info.Extra = make(map[string][]string)
+		}
+		info.Extra[PodNameKey] = []string{sa.PodName}
+		info.Extra[PodUIDKey] = []string{sa.PodUID}
+	}
+	if sa.CredentialID != "" {
+		if info.Extra == nil {
+			info.Extra = make(map[string][]string)
+		}
+		info.Extra[CredentialIDKey] = []string{sa.CredentialID}
+	}
+	if sa.NodeName != "" {
+		if info.Extra == nil {
+			info.Extra = make(map[string][]string)
+		}
+		info.Extra[NodeNameKey] = []string{sa.NodeName}
+		// node UID is optional and will only be set if the node name is set
+		if sa.NodeUID != "" {
+			info.Extra[NodeUIDKey] = []string{sa.NodeUID}
 		}
 	}
+
 	return info
 }
 
+// CredentialIDForJTI converts a given JTI string into a credential identifier for use in a
+// users 'extra' info.
+func CredentialIDForJTI(jti string) string {
+	if len(jti) == 0 {
+		return ""
+	}
+	return "JTI=" + jti
+}
+
 // IsServiceAccountToken returns true if the secret is a valid api token for the service account
 func IsServiceAccountToken(secret *v1.Secret, sa *v1.ServiceAccount) bool {
 	if secret.Type != v1.SecretTypeServiceAccountToken {

vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go

@@ -54,6 +54,7 @@ func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
 		c.AllowCacheTTL,
 		c.DenyCacheTTL,
 		*c.WebhookRetryBackoff,
+		authorizer.DecisionNoOpinion,
 		webhook.AuthorizerMetrics{
 			RecordRequestTotal:   RecordRequestTotal,
 			RecordRequestLatency: RecordRequestLatency,

vendor/k8s.io/apiserver/pkg/authorization/cel/compile.go

@@ -0,0 +1,214 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"fmt"
+
+	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/common/types/ref"
+
+	authorizationv1 "k8s.io/api/authorization/v1"
+	"k8s.io/apimachinery/pkg/util/version"
+	apiservercel "k8s.io/apiserver/pkg/cel"
+	"k8s.io/apiserver/pkg/cel/environment"
+)
+
+const (
+	subjectAccessReviewRequestVarName = "request"
+)
+
+// CompilationResult represents a compiled authorization cel expression.
+type CompilationResult struct {
+	Program            cel.Program
+	ExpressionAccessor ExpressionAccessor
+}
+
+// EvaluationResult contains the minimal required fields and metadata of a cel evaluation
+type EvaluationResult struct {
+	EvalResult         ref.Val
+	ExpressionAccessor ExpressionAccessor
+}
+
+// Compiler is an interface for compiling CEL expressions with the desired environment mode.
+type Compiler interface {
+	CompileCELExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error)
+}
+
+type compiler struct {
+	envSet *environment.EnvSet
+}
+
+// NewCompiler returns a new Compiler.
+func NewCompiler(env *environment.EnvSet) Compiler {
+	return &compiler{
+		envSet: mustBuildEnv(env),
+	}
+}
+
+func (c compiler) CompileCELExpression(expressionAccessor ExpressionAccessor) (CompilationResult, error) {
+	resultError := func(errorString string, errType apiservercel.ErrorType) (CompilationResult, error) {
+		err := &apiservercel.Error{
+			Type:   errType,
+			Detail: errorString,
+		}
+		return CompilationResult{
+			ExpressionAccessor: expressionAccessor,
+		}, err
+	}
+	env, err := c.envSet.Env(environment.StoredExpressions)
+	if err != nil {
+		return resultError(fmt.Sprintf("unexpected error loading CEL environment: %v", err), apiservercel.ErrorTypeInternal)
+	}
+	ast, issues := env.Compile(expressionAccessor.GetExpression())
+	if issues != nil {
+		return resultError("compilation failed: "+issues.String(), apiservercel.ErrorTypeInvalid)
+	}
+	found := false
+	returnTypes := expressionAccessor.ReturnTypes()
+	for _, returnType := range returnTypes {
+		if ast.OutputType() == returnType {
+			found = true
+			break
+		}
+	}
+	if !found {
+		var reason string
+		if len(returnTypes) == 1 {
+			reason = fmt.Sprintf("must evaluate to %v but got %v", returnTypes[0].String(), ast.OutputType())
+		} else {
+			reason = fmt.Sprintf("must evaluate to one of %v", returnTypes)
+		}
+
+		return resultError(reason, apiservercel.ErrorTypeInvalid)
+	}
+	_, err = cel.AstToCheckedExpr(ast)
+	if err != nil {
+		// should be impossible since env.Compile returned no issues
+		return resultError("unexpected compilation error: "+err.Error(), apiservercel.ErrorTypeInternal)
+	}
+	prog, err := env.Program(ast)
+	if err != nil {
+		return resultError("program instantiation failed: "+err.Error(), apiservercel.ErrorTypeInternal)
+	}
+	return CompilationResult{
+		Program:            prog,
+		ExpressionAccessor: expressionAccessor,
+	}, nil
+}
+
+func mustBuildEnv(baseEnv *environment.EnvSet) *environment.EnvSet {
+	field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
+		return apiservercel.NewDeclField(name, declType, required, nil, nil)
+	}
+	fields := func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField {
+		result := make(map[string]*apiservercel.DeclField, len(fields))
+		for _, f := range fields {
+			result[f.Name] = f
+		}
+		return result
+	}
+	subjectAccessReviewSpecRequestType := buildRequestType(field, fields)
+	extended, err := baseEnv.Extend(
+		environment.VersionedOptions{
+			// we record this as 1.0 since it was available in the
+			// first version that supported this feature
+			IntroducedVersion: version.MajorMinor(1, 0),
+			EnvOptions: []cel.EnvOption{
+				cel.Variable(subjectAccessReviewRequestVarName, subjectAccessReviewSpecRequestType.CelType()),
+			},
+			DeclTypes: []*apiservercel.DeclType{
+				subjectAccessReviewSpecRequestType,
+			},
+		},
+	)
+	if err != nil {
+		panic(fmt.Sprintf("environment misconfigured: %v", err))
+	}
+
+	return extended
+}
+
+// buildRequestType generates a DeclType for SubjectAccessReviewSpec.
+// if attributes are added here, also add to convertObjectToUnstructured.
+func buildRequestType(field func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField, fields func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField) *apiservercel.DeclType {
+	resourceAttributesType := buildResourceAttributesType(field, fields)
+	nonResourceAttributesType := buildNonResourceAttributesType(field, fields)
+	return apiservercel.NewObjectType("kubernetes.SubjectAccessReviewSpec", fields(
+		field("resourceAttributes", resourceAttributesType, false),
+		field("nonResourceAttributes", nonResourceAttributesType, false),
+		field("user", apiservercel.StringType, false),
+		field("groups", apiservercel.NewListType(apiservercel.StringType, -1), false),
+		field("extra", apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewListType(apiservercel.StringType, -1), -1), false),
+		field("uid", apiservercel.StringType, false),
+	))
+}
+
+// buildResourceAttributesType generates a DeclType for ResourceAttributes.
+// if attributes are added here, also add to convertObjectToUnstructured.
+func buildResourceAttributesType(field func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField, fields func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField) *apiservercel.DeclType {
+	return apiservercel.NewObjectType("kubernetes.ResourceAttributes", fields(
+		field("namespace", apiservercel.StringType, false),
+		field("verb", apiservercel.StringType, false),
+		field("group", apiservercel.StringType, false),
+		field("version", apiservercel.StringType, false),
+		field("resource", apiservercel.StringType, false),
+		field("subresource", apiservercel.StringType, false),
+		field("name", apiservercel.StringType, false),
+	))
+}
+
+// buildNonResourceAttributesType generates a DeclType for NonResourceAttributes.
+// if attributes are added here, also add to convertObjectToUnstructured.
+func buildNonResourceAttributesType(field func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField, fields func(fields ...*apiservercel.DeclField) map[string]*apiservercel.DeclField) *apiservercel.DeclType {
+	return apiservercel.NewObjectType("kubernetes.NonResourceAttributes", fields(
+		field("path", apiservercel.StringType, false),
+		field("verb", apiservercel.StringType, false),
+	))
+}
+
+func convertObjectToUnstructured(obj *authorizationv1.SubjectAccessReviewSpec) map[string]interface{} {
+	// Construct version containing every SubjectAccessReview user and string attribute field, even omitempty ones, for evaluation by CEL
+	extra := obj.Extra
+	if extra == nil {
+		extra = map[string]authorizationv1.ExtraValue{}
+	}
+	ret := map[string]interface{}{
+		"user":   obj.User,
+		"groups": obj.Groups,
+		"uid":    string(obj.UID),
+		"extra":  extra,
+	}
+	if obj.ResourceAttributes != nil {
+		ret["resourceAttributes"] = map[string]string{
+			"namespace":   obj.ResourceAttributes.Namespace,
+			"verb":        obj.ResourceAttributes.Verb,
+			"group":       obj.ResourceAttributes.Group,
+			"version":     obj.ResourceAttributes.Version,
+			"resource":    obj.ResourceAttributes.Resource,
+			"subresource": obj.ResourceAttributes.Subresource,
+			"name":        obj.ResourceAttributes.Name,
+		}
+	}
+	if obj.NonResourceAttributes != nil {
+		ret["nonResourceAttributes"] = map[string]string{
+			"verb": obj.NonResourceAttributes.Verb,
+			"path": obj.NonResourceAttributes.Path,
+		}
+	}
+	return ret
+}

vendor/k8s.io/apiserver/pkg/authorization/cel/interface.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	celgo "github.com/google/cel-go/cel"
+)
+
+type ExpressionAccessor interface {
+	GetExpression() string
+	ReturnTypes() []*celgo.Type
+}
+
+var _ ExpressionAccessor = &SubjectAccessReviewMatchCondition{}
+
+// SubjectAccessReviewMatchCondition is a CEL expression that maps a SubjectAccessReview request to a list of values.
+type SubjectAccessReviewMatchCondition struct {
+	Expression string
+}
+
+func (v *SubjectAccessReviewMatchCondition) GetExpression() string {
+	return v.Expression
+}
+
+func (v *SubjectAccessReviewMatchCondition) ReturnTypes() []*celgo.Type {
+	return []*celgo.Type{celgo.BoolType}
+}

vendor/k8s.io/apiserver/pkg/authorization/cel/matcher.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cel
+
+import (
+	"context"
+	"fmt"
+
+	celgo "github.com/google/cel-go/cel"
+
+	authorizationv1 "k8s.io/api/authorization/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+type CELMatcher struct {
+	CompilationResults []CompilationResult
+}
+
+// eval evaluates the given SubjectAccessReview against all cel matchCondition expression
+func (c *CELMatcher) Eval(ctx context.Context, r *authorizationv1.SubjectAccessReview) (bool, error) {
+	var evalErrors []error
+	va := map[string]interface{}{
+		"request": convertObjectToUnstructured(&r.Spec),
+	}
+	for _, compilationResult := range c.CompilationResults {
+		evalResult, _, err := compilationResult.Program.ContextEval(ctx, va)
+		if err != nil {
+			evalErrors = append(evalErrors, fmt.Errorf("cel evaluation error: expression '%v' resulted in error: %w", compilationResult.ExpressionAccessor.GetExpression(), err))
+			continue
+		}
+		if evalResult.Type() != celgo.BoolType {
+			evalErrors = append(evalErrors, fmt.Errorf("cel evaluation error: expression '%v' eval result type should be bool but got %W", compilationResult.ExpressionAccessor.GetExpression(), evalResult.Type()))
+			continue
+		}
+		match, ok := evalResult.Value().(bool)
+		if !ok {
+			evalErrors = append(evalErrors, fmt.Errorf("cel evaluation error: expression '%v' eval result value should be bool but got %W", compilationResult.ExpressionAccessor.GetExpression(), evalResult.Value()))
+			continue
+		}
+		// If at least one matchCondition successfully evaluates to FALSE,
+		// return early
+		if !match {
+			return false, nil
+		}
+	}
+	// if there is any error, return
+	if len(evalErrors) > 0 {
+		return false, utilerrors.NewAggregate(evalErrors)
+	}
+	// return ALL matchConditions evaluate to TRUE successfully without error
+	return true, nil
+}

vendor/k8s.io/apiserver/pkg/cel/common/adaptor.go

@@ -56,12 +56,27 @@ type Schema interface {
 
 // Validations contains OpenAPI validation that the CEL library uses.
 type Validations interface {
+	Pattern() string
+	Minimum() *float64
+	IsExclusiveMinimum() bool
+	Maximum() *float64
+	IsExclusiveMaximum() bool
+	MultipleOf() *float64
+	MinItems() *int64
 	MaxItems() *int64
+	MinLength() *int64
 	MaxLength() *int64
+	MinProperties() *int64
 	MaxProperties() *int64
 	Required() []string
 	Enum() []any
 	Nullable() bool
+	UniqueItems() bool
+
+	AllOf() []Schema
+	OneOf() []Schema
+	AnyOf() []Schema
+	Not() Schema
 }
 
 // KubeExtensions contains Kubernetes-specific extensions to the OpenAPI schema.
@@ -71,6 +86,16 @@ type KubeExtensions interface {
 	IsXPreserveUnknownFields() bool
 	XListType() string
 	XListMapKeys() []string
+	XMapType() string
+	XValidations() []ValidationRule
+}
+
+// ValidationRule represents a single x-kubernetes-validations rule.
+type ValidationRule interface {
+	Rule() string
+	Message() string
+	MessageExpression() string
+	FieldPath() string
 }
 
 // SchemaOrBool contains either a schema or a boolean indicating if the object

vendor/k8s.io/apiserver/pkg/cel/common/equality.go

@@ -0,0 +1,334 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"reflect"
+	"time"
+)
+
+// CorrelatedObject represents a node in a tree of objects that are being
+// validated. It is used to keep track of the old value of an object during
+// traversal of the new value. It is also used to cache the results of
+// DeepEqual comparisons between the old and new values of objects.
+//
+// All receiver functions support being called on `nil` to support ergonomic
+// recursive descent. The nil `CorrelatedObject` represents an uncorrelatable
+// node in the tree.
+//
+// CorrelatedObject is not thread-safe. It is the responsibility of the caller
+// to handle concurrency, if any.
+type CorrelatedObject struct {
+	// Currently correlated old value during traversal of the schema/object
+	OldValue interface{}
+
+	// Value being validated
+	Value interface{}
+
+	// Schema used for validation of this value. The schema is also used
+	// to determine how to correlate the old object.
+	Schema Schema
+
+	// Duration spent on ratcheting validation for this object and all of its
+	// children.
+	Duration *time.Duration
+
+	// Scratch space below, may change during validation
+
+	// Cached comparison result of DeepEqual of `value` and `thunk.oldValue`
+	comparisonResult *bool
+
+	// Cached map representation of a map-type list, or nil if not map-type list
+	mapList MapList
+
+	// Children spawned by a call to `Validate` on this object
+	// key is either a string or an index, depending upon whether `value` is
+	// a map or a list, respectively.
+	//
+	// The list of children may be incomplete depending upon if the internal
+	// logic of kube-openapi's SchemaValidator short-circuited before
+	// reaching all of the children.
+	//
+	// It should be expected to have an entry for either all of the children, or
+	// none of them.
+	children map[interface{}]*CorrelatedObject
+}
+
+func NewCorrelatedObject(new, old interface{}, schema Schema) *CorrelatedObject {
+	d := time.Duration(0)
+	return &CorrelatedObject{
+		OldValue: old,
+		Value:    new,
+		Schema:   schema,
+		Duration: &d,
+	}
+}
+
+// If OldValue or Value is not a list, or the index is out of bounds of the
+// Value list, returns nil
+// If oldValue is a list, this considers the x-list-type to decide how to
+// correlate old values:
+//
+// If listType is map, creates a map representation of the list using the designated
+// map-keys, caches it for future calls, and returns the map value, or nil if
+// the correlated key is not in the old map
+//
+// Otherwise, if the list type is not correlatable this funcion returns nil.
+func (r *CorrelatedObject) correlateOldValueForChildAtNewIndex(index int) interface{} {
+	oldAsList, ok := r.OldValue.([]interface{})
+	if !ok {
+		return nil
+	}
+
+	asList, ok := r.Value.([]interface{})
+	if !ok {
+		return nil
+	} else if len(asList) <= index {
+		// Cannot correlate out of bounds index
+		return nil
+	}
+
+	listType := r.Schema.XListType()
+	switch listType {
+	case "map":
+		// Look up keys for this index in current object
+		currentElement := asList[index]
+
+		oldList := r.mapList
+		if oldList == nil {
+			oldList = MakeMapList(r.Schema, oldAsList)
+			r.mapList = oldList
+		}
+		return oldList.Get(currentElement)
+
+	case "set":
+		// Are sets correlatable? Only if the old value equals the current value.
+		// We might be able to support this, but do not currently see a lot
+		// of value
+		// (would allow you to add/remove items from sets with ratcheting but not change them)
+		return nil
+	case "":
+		fallthrough
+	case "atomic":
+		// Atomic lists are the default are not correlatable by item
+		// Ratcheting is not available on a per-index basis
+		return nil
+	default:
+		// Unrecognized list type. Assume non-correlatable.
+		return nil
+	}
+}
+
+// CachedDeepEqual is equivalent to reflect.DeepEqual, but caches the
+// results in the tree of ratchetInvocationScratch objects on the way:
+//
+// For objects and arrays, this function will make a best effort to make
+// use of past DeepEqual checks performed by this Node's children, if available.
+//
+// If a lazy computation could not be found for all children possibly due
+// to validation logic short circuiting and skipping the children, then
+// this function simply defers to reflect.DeepEqual.
+func (r *CorrelatedObject) CachedDeepEqual() (res bool) {
+	start := time.Now()
+	defer func() {
+		if r != nil && r.Duration != nil {
+			*r.Duration += time.Since(start)
+		}
+	}()
+
+	if r == nil {
+		// Uncorrelatable node is not considered equal to its old value
+		return false
+	} else if r.comparisonResult != nil {
+		return *r.comparisonResult
+	}
+
+	defer func() {
+		r.comparisonResult = &res
+	}()
+
+	if r.Value == nil && r.OldValue == nil {
+		return true
+	} else if r.Value == nil || r.OldValue == nil {
+		return false
+	}
+
+	oldAsArray, oldIsArray := r.OldValue.([]interface{})
+	newAsArray, newIsArray := r.Value.([]interface{})
+
+	oldAsMap, oldIsMap := r.OldValue.(map[string]interface{})
+	newAsMap, newIsMap := r.Value.(map[string]interface{})
+
+	// If old and new are not the same type, they are not equal
+	if (oldIsArray != newIsArray) || oldIsMap != newIsMap {
+		return false
+	}
+
+	// Objects are known to be same type of (map, slice, or primitive)
+	switch {
+	case oldIsArray:
+		// Both arrays case. oldIsArray == newIsArray
+		if len(oldAsArray) != len(newAsArray) {
+			return false
+		}
+
+		for i := range newAsArray {
+			child := r.Index(i)
+			if child == nil {
+				if r.mapList == nil {
+					// Treat non-correlatable array as a unit with reflect.DeepEqual
+					return reflect.DeepEqual(oldAsArray, newAsArray)
+				}
+
+				// If array is correlatable, but old not found. Just short circuit
+				// comparison
+				return false
+
+			} else if !child.CachedDeepEqual() {
+				// If one child is not equal the entire object is not equal
+				return false
+			}
+		}
+
+		return true
+	case oldIsMap:
+		// Both maps case. oldIsMap == newIsMap
+		if len(oldAsMap) != len(newAsMap) {
+			return false
+		}
+
+		for k := range newAsMap {
+			child := r.Key(k)
+			if child == nil {
+				// Un-correlatable child due to key change.
+				// Objects are not equal.
+				return false
+			} else if !child.CachedDeepEqual() {
+				// If one child is not equal the entire object is not equal
+				return false
+			}
+		}
+
+		return true
+
+	default:
+		// Primitive: use reflect.DeepEqual
+		return reflect.DeepEqual(r.OldValue, r.Value)
+	}
+}
+
+// Key returns the child of the receiver with the given name.
+// Returns nil if the given name is does not exist in the new object, or its
+// value is not correlatable to an old value.
+// If receiver is nil or if the new value is not an object/map, returns nil.
+func (r *CorrelatedObject) Key(field string) *CorrelatedObject {
+	start := time.Now()
+	defer func() {
+		if r != nil && r.Duration != nil {
+			*r.Duration += time.Since(start)
+		}
+	}()
+
+	if r == nil || r.Schema == nil {
+		return nil
+	} else if existing, exists := r.children[field]; exists {
+		return existing
+	}
+
+	// Find correlated old value
+	oldAsMap, okOld := r.OldValue.(map[string]interface{})
+	newAsMap, okNew := r.Value.(map[string]interface{})
+	if !okOld || !okNew {
+		return nil
+	}
+
+	oldValueForField, okOld := oldAsMap[field]
+	newValueForField, okNew := newAsMap[field]
+	if !okOld || !okNew {
+		return nil
+	}
+
+	var propertySchema Schema
+	if prop, exists := r.Schema.Properties()[field]; exists {
+		propertySchema = prop
+	} else if addP := r.Schema.AdditionalProperties(); addP != nil && addP.Schema() != nil {
+		propertySchema = addP.Schema()
+	} else {
+		return nil
+	}
+
+	if r.children == nil {
+		r.children = make(map[interface{}]*CorrelatedObject, len(newAsMap))
+	}
+
+	res := &CorrelatedObject{
+		OldValue: oldValueForField,
+		Value:    newValueForField,
+		Schema:   propertySchema,
+		Duration: r.Duration,
+	}
+	r.children[field] = res
+	return res
+}
+
+// Index returns the child of the receiver at the given index.
+// Returns nil if the given index is out of bounds, or its value is not
+// correlatable to an old value.
+// If receiver is nil or if the new value is not an array, returns nil.
+func (r *CorrelatedObject) Index(i int) *CorrelatedObject {
+	start := time.Now()
+	defer func() {
+		if r != nil && r.Duration != nil {
+			*r.Duration += time.Since(start)
+		}
+	}()
+
+	if r == nil || r.Schema == nil {
+		return nil
+	} else if existing, exists := r.children[i]; exists {
+		return existing
+	}
+
+	asList, ok := r.Value.([]interface{})
+	if !ok || len(asList) <= i {
+		return nil
+	}
+
+	oldValueForIndex := r.correlateOldValueForChildAtNewIndex(i)
+	if oldValueForIndex == nil {
+		return nil
+	}
+	var itemSchema Schema
+	if i := r.Schema.Items(); i != nil {
+		itemSchema = i
+	} else {
+		return nil
+	}
+
+	if r.children == nil {
+		r.children = make(map[interface{}]*CorrelatedObject, len(asList))
+	}
+
+	res := &CorrelatedObject{
+		OldValue: oldValueForIndex,
+		Value:    asList[i],
+		Schema:   itemSchema,
+		Duration: r.Duration,
+	}
+	r.children[i] = res
+	return res
+}

vendor/k8s.io/apiserver/pkg/cel/common/schemas.go

@@ -165,7 +165,11 @@ func SchemaDeclType(s Schema, isResourceRoot bool) *apiservercel.DeclType {
 			// unicode code point can be up to 4 bytes long)
 			strWithMaxLength.MaxElements = zeroIfNegative(*s.MaxLength()) * 4
 		} else {
-			strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
+			if len(s.Enum()) > 0 {
+				strWithMaxLength.MaxElements = estimateMaxStringEnumLength(s)
+			} else {
+				strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s)
+			}
 		}
 		return strWithMaxLength
 	case "boolean":
@@ -239,6 +243,19 @@ func estimateMaxStringLengthPerRequest(s Schema) int64 {
 	}
 }
 
+// estimateMaxStringLengthPerRequest estimates the maximum string length (in characters)
+// that has a set of enum values.
+// The result of the estimation is the length of the longest possible value.
+func estimateMaxStringEnumLength(s Schema) int64 {
+	var maxLength int64
+	for _, v := range s.Enum() {
+		if s, ok := v.(string); ok && int64(len(s)) > maxLength {
+			maxLength = int64(len(s))
+		}
+	}
+	return maxLength
+}
+
 // estimateMaxArrayItemsPerRequest estimates the maximum number of array items with
 // the provided minimum serialized size that can fit into a single request.
 func estimateMaxArrayItemsFromMinSize(minSize int64) int64 {

vendor/k8s.io/apiserver/pkg/cel/common/values.go

@@ -84,18 +84,22 @@ func UnstructuredToVal(unstructured interface{}, schema Schema) ref.Val {
 				},
 			}
 		}
-		// A object with x-kubernetes-preserve-unknown-fields but no properties or additionalProperties is treated
-		// as an empty object.
-		if schema.IsXPreserveUnknownFields() {
-			return &unstructuredMap{
-				value:  m,
-				schema: schema,
-				propSchema: func(key string) (Schema, bool) {
-					return nil, false
-				},
-			}
+
+		// properties and additionalProperties are mutual exclusive, but nothing prevents the situation
+		// where both are missing.
+		// An object that (1) has no properties (2) has no additionalProperties or additionalProperties == false
+		// is treated as an empty object.
+		// An object that has additionalProperties == true is treated as an unstructured map.
+		// An object that has x-kubernetes-preserve-unknown-field extension set is treated as an unstructured map.
+		// Empty object vs unstructured map is differentiated by unstructuredMap implementation with the set schema.
+		// The resulting result remains the same.
+		return &unstructuredMap{
+			value:  m,
+			schema: schema,
+			propSchema: func(key string) (Schema, bool) {
+				return nil, false
+			},
 		}
-		return types.NewErr("invalid object type, expected either Properties or AdditionalProperties with Allows=true and non-empty Schema")
 	}
 
 	if schema.Type() == "array" {

vendor/k8s.io/apiserver/pkg/cel/environment/base.go

@@ -22,7 +22,9 @@ import (
 	"sync"
 
 	"github.com/google/cel-go/cel"
+	"github.com/google/cel-go/checker"
 	"github.com/google/cel-go/ext"
+	"github.com/google/cel-go/interpreter"
 	"golang.org/x/sync/singleflight"
 
 	"k8s.io/apimachinery/pkg/util/version"
@@ -41,7 +43,7 @@ import (
 // desirable because it means that CEL expressions are portable across a wider range
 // of Kubernetes versions.
 func DefaultCompatibilityVersion() *version.Version {
-	return version.MajorMinor(1, 27)
+	return version.MajorMinor(1, 28)
 }
 
 var baseOpts = []VersionedOptions{
@@ -57,7 +59,6 @@ var baseOpts = []VersionedOptions{
 			cel.EagerlyValidateDeclarations(true),
 			cel.DefaultUTCTimeZone(true),
 
-			ext.Strings(ext.StringsVersion(0)),
 			library.URLs(),
 			library.Regex(),
 			library.Lists(),
@@ -81,7 +82,47 @@ var baseOpts = []VersionedOptions{
 			library.Quantity(),
 		},
 	},
-	// TODO: switch to ext.Strings version 2 once format() is fixed to work with HomogeneousAggregateLiterals.
+	// add the new validator in 1.29
+	{
+		IntroducedVersion: version.MajorMinor(1, 29),
+		EnvOptions: []cel.EnvOption{
+			cel.ASTValidators(
+				cel.ValidateDurationLiterals(),
+				cel.ValidateTimestampLiterals(),
+				cel.ValidateRegexLiterals(),
+				cel.ValidateHomogeneousAggregateLiterals(),
+			),
+		},
+	},
+	// String library
+	{
+		IntroducedVersion: version.MajorMinor(1, 0),
+		RemovedVersion:    version.MajorMinor(1, 29),
+		EnvOptions: []cel.EnvOption{
+			ext.Strings(ext.StringsVersion(0)),
+		},
+	},
+	{
+		IntroducedVersion: version.MajorMinor(1, 29),
+		EnvOptions: []cel.EnvOption{
+			ext.Strings(ext.StringsVersion(2)),
+		},
+	},
+	// Set library
+	{
+		IntroducedVersion: version.MajorMinor(1, 29),
+		EnvOptions: []cel.EnvOption{
+			ext.Sets(),
+			// cel-go v0.17.7 introduced CostEstimatorOptions.
+			// Previous the presence has a cost of 0 but cel fixed it to 1. We still set to 0 here to avoid breaking changes.
+			cel.CostEstimatorOptions(checker.PresenceTestHasCost(false)),
+		},
+		ProgramOptions: []cel.ProgramOption{
+			// cel-go v0.17.7 introduced CostTrackerOptions.
+			// Previous the presence has a cost of 0 but cel fixed it to 1. We still set to 0 here to avoid breaking changes.
+			cel.CostTrackerOptions(interpreter.PresenceTestHasCost(false)),
+		},
+	},
 }
 
 // MustBaseEnvSet returns the common CEL base environments for Kubernetes for Version, or panics

vendor/k8s.io/apiserver/pkg/cel/lazy/lazy.go

@@ -35,7 +35,7 @@ var _ traits.Mapper = (*MapValue)(nil)
 // MapValue is a map that lazily evaluate its value when a field is first accessed.
 // The map value is not designed to be thread-safe.
 type MapValue struct {
-	typeValue *types.TypeValue
+	typeValue *types.Type
 
 	// values are previously evaluated values obtained from callbacks
 	values map[string]ref.Val

vendor/k8s.io/apiserver/pkg/cel/library/authz.go

@@ -202,6 +202,10 @@ var authzLib = &authz{}
 
 type authz struct{}
 
+func (*authz) LibraryName() string {
+	return "k8s.authz"
+}
+
 var authzLibraryDecls = map[string][]cel.FunctionOpt{
 	"path": {
 		cel.MemberOverload("authorizer_path", []*cel.Type{AuthorizerType, cel.StringType}, PathCheckType,
@@ -578,7 +582,7 @@ type decisionVal struct {
 // any object type that has receiver functions but does not expose any fields to
 // CEL.
 type receiverOnlyObjectVal struct {
-	typeValue *types.TypeValue
+	typeValue *types.Type
 }
 
 // receiverOnlyVal returns a receiverOnlyObjectVal for the given type.

vendor/k8s.io/apiserver/pkg/cel/library/cost.go

@@ -101,8 +101,8 @@ func (l *CostEstimator) EstimateCallCost(function, overloadId string, target *ch
 			// If the list contains strings or bytes, add the cost of traversing all the strings/bytes as a way
 			// of estimating the additional comparison cost.
 			if elNode := l.listElementNode(*target); elNode != nil {
-				t := elNode.Type().GetPrimitive()
-				if t == exprpb.Type_STRING || t == exprpb.Type_BYTES {
+				k := elNode.Type().Kind()
+				if k == types.StringKind || k == types.BytesKind {
 					sz := l.sizeEstimate(elNode)
 					elCost = elCost.Add(sz.MultiplyByCostFactor(common.StringTraversalCostFactor))
 				}
@@ -247,7 +247,8 @@ func (l *CostEstimator) sizeEstimate(t checker.AstNode) checker.SizeEstimate {
 }
 
 func (l *CostEstimator) listElementNode(list checker.AstNode) checker.AstNode {
-	if lt := list.Type().GetListType(); lt != nil {
+	if params := list.Type().Parameters(); len(params) > 0 {
+		lt := params[0]
 		nodePath := list.Path()
 		if nodePath != nil {
 			// Provide path if we have it so that a OpenAPIv3 maxLength validation can be looked up, if it exists
@@ -255,10 +256,10 @@ func (l *CostEstimator) listElementNode(list checker.AstNode) checker.AstNode {
 			path := make([]string, len(nodePath)+1)
 			copy(path, nodePath)
 			path[len(nodePath)] = "@items"
-			return &itemsNode{path: path, t: lt.GetElemType(), expr: nil}
+			return &itemsNode{path: path, t: lt, expr: nil}
 		} else {
 			// Provide just the type if no path is available so that worst case size can be looked up based on type.
-			return &itemsNode{t: lt.GetElemType(), expr: nil}
+			return &itemsNode{t: lt, expr: nil}
 		}
 	}
 	return nil
@@ -273,7 +274,7 @@ func (l *CostEstimator) EstimateSize(element checker.AstNode) *checker.SizeEstim
 
 type itemsNode struct {
 	path []string
-	t    *exprpb.Type
+	t    *types.Type
 	expr *exprpb.Expr
 }
 
@@ -281,7 +282,7 @@ func (i *itemsNode) Path() []string {
 	return i.path
 }
 
-func (i *itemsNode) Type() *exprpb.Type {
+func (i *itemsNode) Type() *types.Type {
 	return i.t
 }
 

vendor/k8s.io/apiserver/pkg/cel/library/lists.go

@@ -95,6 +95,10 @@ var listsLib = &lists{}
 
 type lists struct{}
 
+func (*lists) LibraryName() string {
+	return "k8s.lists"
+}
+
 var paramA = cel.TypeParamType("A")
 
 // CEL typeParams can be used to constraint to a specific trait (e.g. traits.ComparableType) if the 1st operand is the type to constrain.

vendor/k8s.io/apiserver/pkg/cel/library/quantity.go

@@ -22,6 +22,7 @@ import (
 	"github.com/google/cel-go/cel"
 	"github.com/google/cel-go/common/types"
 	"github.com/google/cel-go/common/types/ref"
+
 	"k8s.io/apimachinery/pkg/api/resource"
 	apiservercel "k8s.io/apiserver/pkg/cel"
 )
@@ -141,6 +142,10 @@ var quantityLib = &quantity{}
 
 type quantity struct{}
 
+func (*quantity) LibraryName() string {
+	return "k8s.quantity"
+}
+
 var quantityLibraryDecls = map[string][]cel.FunctionOpt{
 	"quantity": {
 		cel.Overload("string_to_quantity", []*cel.Type{cel.StringType}, apiservercel.QuantityType, cel.UnaryBinding((stringToQuantity))),

vendor/k8s.io/apiserver/pkg/cel/library/regex.go

@@ -51,6 +51,10 @@ var regexLib = &regex{}
 
 type regex struct{}
 
+func (*regex) LibraryName() string {
+	return "k8s.regex"
+}
+
 var regexLibraryDecls = map[string][]cel.FunctionOpt{
 	"find": {
 		cel.MemberOverload("string_find_string", []*cel.Type{cel.StringType, cel.StringType}, cel.StringType,

vendor/k8s.io/apiserver/pkg/cel/library/test.go

@@ -37,6 +37,10 @@ type testLib struct {
 	version uint32
 }
 
+func (*testLib) LibraryName() string {
+	return "k8s.test"
+}
+
 type TestOption func(*testLib) *testLib
 
 func TestVersion(version uint32) func(lib *testLib) *testLib {

vendor/k8s.io/apiserver/pkg/cel/library/urls.go

@@ -112,6 +112,10 @@ var urlsLib = &urls{}
 
 type urls struct{}
 
+func (*urls) LibraryName() string {
+	return "k8s.urls"
+}
+
 var urlLibraryDecls = map[string][]cel.FunctionOpt{
 	"url": {
 		cel.Overload("string_to_url", []*cel.Type{cel.StringType}, apiservercel.URLType,

vendor/k8s.io/apiserver/pkg/cel/openapi/adaptor.go

@@ -54,6 +54,10 @@ func (s *Schema) Format() string {
 	return s.Schema.Format
 }
 
+func (s *Schema) Pattern() string {
+	return s.Schema.Pattern
+}
+
 func (s *Schema) Items() common.Schema {
 	if s.Schema.Items == nil || s.Schema.Items.Schema == nil {
 		return nil
@@ -86,14 +90,50 @@ func (s *Schema) Default() any {
 	return s.Schema.Default
 }
 
+func (s *Schema) Minimum() *float64 {
+	return s.Schema.Minimum
+}
+
+func (s *Schema) IsExclusiveMinimum() bool {
+	return s.Schema.ExclusiveMinimum
+}
+
+func (s *Schema) Maximum() *float64 {
+	return s.Schema.Maximum
+}
+
+func (s *Schema) IsExclusiveMaximum() bool {
+	return s.Schema.ExclusiveMaximum
+}
+
+func (s *Schema) MultipleOf() *float64 {
+	return s.Schema.MultipleOf
+}
+
+func (s *Schema) UniqueItems() bool {
+	return s.Schema.UniqueItems
+}
+
+func (s *Schema) MinItems() *int64 {
+	return s.Schema.MinItems
+}
+
 func (s *Schema) MaxItems() *int64 {
 	return s.Schema.MaxItems
 }
 
+func (s *Schema) MinLength() *int64 {
+	return s.Schema.MinLength
+}
+
 func (s *Schema) MaxLength() *int64 {
 	return s.Schema.MaxLength
 }
 
+func (s *Schema) MinProperties() *int64 {
+	return s.Schema.MinProperties
+}
+
 func (s *Schema) MaxProperties() *int64 {
 	return s.Schema.MaxProperties
 }
@@ -110,6 +150,40 @@ func (s *Schema) Nullable() bool {
 	return s.Schema.Nullable
 }
 
+func (s *Schema) AllOf() []common.Schema {
+	var res []common.Schema
+	for _, nestedSchema := range s.Schema.AllOf {
+		nestedSchema := nestedSchema
+		res = append(res, &Schema{&nestedSchema})
+	}
+	return res
+}
+
+func (s *Schema) AnyOf() []common.Schema {
+	var res []common.Schema
+	for _, nestedSchema := range s.Schema.AnyOf {
+		nestedSchema := nestedSchema
+		res = append(res, &Schema{&nestedSchema})
+	}
+	return res
+}
+
+func (s *Schema) OneOf() []common.Schema {
+	var res []common.Schema
+	for _, nestedSchema := range s.Schema.OneOf {
+		nestedSchema := nestedSchema
+		res = append(res, &Schema{&nestedSchema})
+	}
+	return res
+}
+
+func (s *Schema) Not() common.Schema {
+	if s.Schema.Not == nil {
+		return nil
+	}
+	return &Schema{s.Schema.Not}
+}
+
 func (s *Schema) IsXIntOrString() bool {
 	return isXIntOrString(s.Schema)
 }
@@ -126,10 +200,18 @@ func (s *Schema) XListType() string {
 	return getXListType(s.Schema)
 }
 
+func (s *Schema) XMapType() string {
+	return getXMapType(s.Schema)
+}
+
 func (s *Schema) XListMapKeys() []string {
 	return getXListMapKeys(s.Schema)
 }
 
+func (s *Schema) XValidations() []common.ValidationRule {
+	return getXValidations(s.Schema)
+}
+
 func (s *Schema) WithTypeAndObjectMeta() common.Schema {
 	return &Schema{common.WithTypeAndObjectMeta(s.Schema)}
 }

vendor/k8s.io/apiserver/pkg/cel/openapi/extensions.go

@@ -18,6 +18,7 @@ package openapi
 
 import (
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apiserver/pkg/cel/common"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 )
 
@@ -47,6 +48,11 @@ func getXListType(schema *spec.Schema) string {
 	return s
 }
 
+func getXMapType(schema *spec.Schema) string {
+	s, _ := schema.Extensions.GetString(extMapType)
+	return s
+}
+
 func getXListMapKeys(schema *spec.Schema) []string {
 	mapKeys, ok := schema.Extensions.GetStringSlice(extListMapKeys)
 	if !ok {
@@ -55,8 +61,47 @@ func getXListMapKeys(schema *spec.Schema) []string {
 	return mapKeys
 }
 
+type ValidationRule struct {
+	RuleField              string `json:"rule"`
+	MessageField           string `json:"message"`
+	MessageExpressionField string `json:"messageExpression"`
+	PathField              string `json:"fieldPath"`
+}
+
+func (v ValidationRule) Rule() string {
+	return v.RuleField
+}
+
+func (v ValidationRule) Message() string {
+	return v.MessageField
+}
+
+func (v ValidationRule) FieldPath() string {
+	return v.PathField
+}
+
+func (v ValidationRule) MessageExpression() string {
+	return v.MessageExpressionField
+}
+
+// TODO: simplify
+func getXValidations(schema *spec.Schema) []common.ValidationRule {
+	var rules []ValidationRule
+	err := schema.Extensions.GetObject(extValidations, &rules)
+	if err != nil {
+		return nil
+	}
+	results := make([]common.ValidationRule, len(rules))
+	for i, rule := range rules {
+		results[i] = rule
+	}
+	return results
+}
+
 const extIntOrString = "x-kubernetes-int-or-string"
 const extEmbeddedResource = "x-kubernetes-embedded-resource"
 const extPreserveUnknownFields = "x-kubernetes-preserve-unknown-fields"
 const extListType = "x-kubernetes-list-type"
+const extMapType = "x-kubernetes-map-type"
 const extListMapKeys = "x-kubernetes-list-map-keys"
+const extValidations = "x-kubernetes-validations"

vendor/k8s.io/apiserver/pkg/cel/openapi/resolver/combined.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package resolver
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kube-openapi/pkg/validation/spec"
+)
+
+// Combine combines the DefinitionsSchemaResolver with a secondary schema resolver.
+// The resulting schema resolver uses the DefinitionsSchemaResolver for a GVK that DefinitionsSchemaResolver knows,
+// and the secondary otherwise.
+func (d *DefinitionsSchemaResolver) Combine(secondary SchemaResolver) SchemaResolver {
+	return &combinedSchemaResolver{definitions: d, secondary: secondary}
+}
+
+type combinedSchemaResolver struct {
+	definitions *DefinitionsSchemaResolver
+	secondary   SchemaResolver
+}
+
+// ResolveSchema takes a GroupVersionKind (GVK) and returns the OpenAPI schema
+// identified by the GVK.
+// If the DefinitionsSchemaResolver knows the gvk, the DefinitionsSchemaResolver handles the resolution,
+// otherwise, the secondary does.
+func (r *combinedSchemaResolver) ResolveSchema(gvk schema.GroupVersionKind) (*spec.Schema, error) {
+	if _, ok := r.definitions.gvkToRef[gvk]; ok {
+		return r.definitions.ResolveSchema(gvk)
+	}
+	return r.secondary.ResolveSchema(gvk)
+}

vendor/k8s.io/apiserver/pkg/cel/openapi/resolver/definitions.go

@@ -29,40 +29,39 @@ import (
 // DefinitionsSchemaResolver resolves the schema of a built-in type
 // by looking up the OpenAPI definitions.
 type DefinitionsSchemaResolver struct {
-	defs        map[string]common.OpenAPIDefinition
-	gvkToSchema map[schema.GroupVersionKind]*spec.Schema
+	defs     map[string]common.OpenAPIDefinition
+	gvkToRef map[schema.GroupVersionKind]string
 }
 
 // NewDefinitionsSchemaResolver creates a new DefinitionsSchemaResolver.
 // An example working setup:
-// scheme         = "k8s.io/client-go/kubernetes/scheme".Scheme
 // getDefinitions = "k8s.io/kubernetes/pkg/generated/openapi".GetOpenAPIDefinitions
-func NewDefinitionsSchemaResolver(scheme *runtime.Scheme, getDefinitions common.GetOpenAPIDefinitions) *DefinitionsSchemaResolver {
-	gvkToSchema := make(map[schema.GroupVersionKind]*spec.Schema)
-	namer := openapi.NewDefinitionNamer(scheme)
+// scheme         = "k8s.io/client-go/kubernetes/scheme".Scheme
+func NewDefinitionsSchemaResolver(getDefinitions common.GetOpenAPIDefinitions, schemes ...*runtime.Scheme) *DefinitionsSchemaResolver {
+	gvkToRef := make(map[schema.GroupVersionKind]string)
+	namer := openapi.NewDefinitionNamer(schemes...)
 	defs := getDefinitions(func(path string) spec.Ref {
 		return spec.MustCreateRef(path)
 	})
-	for name, def := range defs {
+	for name := range defs {
 		_, e := namer.GetDefinitionName(name)
 		gvks := extensionsToGVKs(e)
-		s := def.Schema // map value not addressable, make copy
 		for _, gvk := range gvks {
-			gvkToSchema[gvk] = &s
+			gvkToRef[gvk] = name
 		}
 	}
 	return &DefinitionsSchemaResolver{
-		gvkToSchema: gvkToSchema,
-		defs:        defs,
+		gvkToRef: gvkToRef,
+		defs:     defs,
 	}
 }
 
 func (d *DefinitionsSchemaResolver) ResolveSchema(gvk schema.GroupVersionKind) (*spec.Schema, error) {
-	s, ok := d.gvkToSchema[gvk]
+	ref, ok := d.gvkToRef[gvk]
 	if !ok {
 		return nil, fmt.Errorf("cannot resolve %v: %w", gvk, ErrSchemaNotFound)
 	}
-	s, err := populateRefs(func(ref string) (*spec.Schema, bool) {
+	s, err := PopulateRefs(func(ref string) (*spec.Schema, bool) {
 		// find the schema by the ref string, and return a deep copy
 		def, ok := d.defs[ref]
 		if !ok {
@@ -70,7 +69,7 @@ func (d *DefinitionsSchemaResolver) ResolveSchema(gvk schema.GroupVersionKind) (
 		}
 		s := def.Schema
 		return &s, true
-	}, s)
+	}, ref)
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/apiserver/pkg/cel/openapi/resolver/discovery.go

@@ -53,34 +53,34 @@ func (r *ClientDiscoveryResolver) ResolveSchema(gvk schema.GroupVersionKind) (*s
 	if err != nil {
 		return nil, err
 	}
-	s, err := resolveType(resp, gvk)
+	ref, err := resolveRef(resp, gvk)
 	if err != nil {
 		return nil, err
 	}
-	s, err = populateRefs(func(ref string) (*spec.Schema, bool) {
+	s, err := PopulateRefs(func(ref string) (*spec.Schema, bool) {
 		s, ok := resp.Components.Schemas[strings.TrimPrefix(ref, refPrefix)]
 		return s, ok
-	}, s)
+	}, ref)
 	if err != nil {
 		return nil, err
 	}
 	return s, nil
 }
 
-func resolveType(resp *schemaResponse, gvk schema.GroupVersionKind) (*spec.Schema, error) {
-	for _, s := range resp.Components.Schemas {
+func resolveRef(resp *schemaResponse, gvk schema.GroupVersionKind) (string, error) {
+	for ref, s := range resp.Components.Schemas {
 		var gvks []schema.GroupVersionKind
 		err := s.Extensions.GetObject(extGVK, &gvks)
 		if err != nil {
-			return nil, err
+			return "", err
 		}
 		for _, g := range gvks {
 			if g == gvk {
-				return s, nil
+				return ref, nil
 			}
 		}
 	}
-	return nil, fmt.Errorf("cannot resolve group version kind %q: %w", gvk, ErrSchemaNotFound)
+	return "", fmt.Errorf("cannot resolve group version kind %q: %w", gvk, ErrSchemaNotFound)
 }
 
 func resourcePathFromGV(gv schema.GroupVersion) string {

vendor/k8s.io/apiserver/pkg/cel/openapi/resolver/refs.go

@@ -19,19 +19,41 @@ package resolver
 import (
 	"fmt"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 )
 
-// populateRefs recursively replaces Refs in the schema with the referred one.
+// PopulateRefs recursively replaces Refs in the schema with the referred one.
 // schemaOf is the callback to find the corresponding schema by the ref.
 // This function will not mutate the original schema. If the schema needs to be
 // mutated, a copy will be returned, otherwise it returns the original schema.
-func populateRefs(schemaOf func(ref string) (*spec.Schema, bool), schema *spec.Schema) (*spec.Schema, error) {
+func PopulateRefs(schemaOf func(ref string) (*spec.Schema, bool), rootRef string) (*spec.Schema, error) {
+	visitedRefs := sets.New[string]()
+	rootSchema, ok := schemaOf(rootRef)
+	visitedRefs.Insert(rootRef)
+	if !ok {
+		return nil, fmt.Errorf("internal error: cannot resolve Ref for root schema %q: %w", rootRef, ErrSchemaNotFound)
+	}
+	return populateRefs(schemaOf, visitedRefs, rootSchema)
+}
+
+func populateRefs(schemaOf func(ref string) (*spec.Schema, bool), visited sets.Set[string], schema *spec.Schema) (*spec.Schema, error) {
 	result := *schema
 	changed := false
 
 	ref, isRef := refOf(schema)
 	if isRef {
+		if visited.Has(ref) {
+			return &spec.Schema{
+				// for circular ref, return an empty object as placeholder
+				SchemaProps: spec.SchemaProps{Type: []string{"object"}},
+			}, nil
+		}
+		visited.Insert(ref)
+		// restore visited state at the end of the recursion.
+		defer func() {
+			visited.Delete(ref)
+		}()
 		// replace the whole schema with the referred one.
 		resolved, ok := schemaOf(ref)
 		if !ok {
@@ -44,7 +66,7 @@ func populateRefs(schemaOf func(ref string) (*spec.Schema, bool), schema *spec.S
 	props := make(map[string]spec.Schema, len(schema.Properties))
 	propsChanged := false
 	for name, prop := range result.Properties {
-		populated, err := populateRefs(schemaOf, &prop)
+		populated, err := populateRefs(schemaOf, visited, &prop)
 		if err != nil {
 			return nil, err
 		}
@@ -58,7 +80,7 @@ func populateRefs(schemaOf func(ref string) (*spec.Schema, bool), schema *spec.S
 		result.Properties = props
 	}
 	if result.AdditionalProperties != nil && result.AdditionalProperties.Schema != nil {
-		populated, err := populateRefs(schemaOf, result.AdditionalProperties.Schema)
+		populated, err := populateRefs(schemaOf, visited, result.AdditionalProperties.Schema)
 		if err != nil {
 			return nil, err
 		}
@@ -69,7 +91,7 @@ func populateRefs(schemaOf func(ref string) (*spec.Schema, bool), schema *spec.S
 	}
 	// schema is a list, populate its items
 	if result.Items != nil && result.Items.Schema != nil {
-		populated, err := populateRefs(schemaOf, result.Items.Schema)
+		populated, err := populateRefs(schemaOf, visited, result.Items.Schema)
 		if err != nil {
 			return nil, err
 		}

vendor/k8s.io/apiserver/pkg/endpoints/discovery/aggregated/handler.go

@@ -229,7 +229,6 @@ func (rdm *resourceDiscoveryManager) AddGroupVersion(source Source, groupName st
 }
 
 func (rdm *resourceDiscoveryManager) addGroupVersionLocked(source Source, groupName string, value apidiscoveryv2beta1.APIVersionDiscovery) {
-	klog.Infof("Adding GroupVersion %s %s to ResourceManager", groupName, value.Version)
 
 	if rdm.apiGroups == nil {
 		rdm.apiGroups = make(map[groupKey]*apidiscoveryv2beta1.APIGroupDiscovery)
@@ -273,6 +272,7 @@ func (rdm *resourceDiscoveryManager) addGroupVersionLocked(source Source, groupN
 		}
 		rdm.apiGroups[key] = group
 	}
+	klog.Infof("Adding GroupVersion %s %s to ResourceManager", groupName, value.Version)
 
 	gv := metav1.GroupVersion{Group: groupName, Version: value.Version}
 	gvKey := groupVersionKey{

vendor/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go

@@ -164,7 +164,7 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
 		req = req.WithContext(request.WithUser(ctx, newUser))
 
 		oldUser, _ := request.UserFrom(ctx)
-		httplog.LogOf(req, w).Addf("%v is acting as %v", oldUser, newUser)
+		httplog.LogOf(req, w).Addf("%v is impersonating %v", userString(oldUser), userString(newUser))
 
 		ae := audit.AuditEventFrom(ctx)
 		audit.LogImpersonatedUser(ae, newUser)
@@ -183,6 +183,24 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
 	})
 }
 
+func userString(u user.Info) string {
+	if u == nil {
+		return "<none>"
+	}
+	b := strings.Builder{}
+	if name := u.GetName(); name == "" {
+		b.WriteString("<empty>")
+	} else {
+		b.WriteString(name)
+	}
+	if groups := u.GetGroups(); len(groups) > 0 {
+		b.WriteString("[")
+		b.WriteString(strings.Join(groups, ","))
+		b.WriteString("]")
+	}
+	return b.String()
+}
+
 func unescapeExtraKey(encodedKey string) string {
 	key, err := url.PathUnescape(encodedKey) // Decode %-encoded bytes.
 	if err != nil {

vendor/k8s.io/apiserver/pkg/endpoints/filters/traces.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"go.opentelemetry.io/otel/trace"
 
 	tracing "k8s.io/component-base/tracing"
@@ -32,7 +33,15 @@ func WithTracing(handler http.Handler, tp trace.TracerProvider) http.Handler {
 		otelhttp.WithPublicEndpoint(),
 		otelhttp.WithTracerProvider(tp),
 	}
+	wrappedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Add the http.target attribute to the otelhttp span
+		// Workaround for https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3743
+		if r.URL != nil {
+			trace.SpanFromContext(r.Context()).SetAttributes(semconv.HTTPTarget(r.URL.RequestURI()))
+		}
+		handler.ServeHTTP(w, r)
+	})
 	// With Noop TracerProvider, the otelhttp still handles context propagation.
 	// See https://github.com/open-telemetry/opentelemetry-go/tree/main/example/passthrough
-	return otelhttp.NewHandler(handler, "KubernetesAPI", opts...)
+	return otelhttp.NewHandler(wrappedHandler, "KubernetesAPI", opts...)
 }

vendor/k8s.io/apiserver/pkg/endpoints/handlers/get.go

@@ -267,7 +267,7 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
 			}
 			requestInfo, _ := request.RequestInfoFrom(ctx)
 			metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() {
-				serveWatch(watcher, scope, outputMediaType, req, w, timeout)
+				serveWatch(watcher, scope, outputMediaType, req, w, timeout, metrics.CleanListScope(ctx, &opts))
 			})
 			return
 		}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/helpers.go

@@ -77,6 +77,96 @@ func (lazy *lazyAccept) String() string {
 	return "unknown"
 }
 
+// lazyAPIGroup implements String() string and it will
+// lazily get Group from request info.
+type lazyAPIGroup struct {
+	req *http.Request
+}
+
+func (lazy *lazyAPIGroup) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.APIGroup
+		}
+	}
+
+	return "unknown"
+}
+
+// lazyAPIVersion implements String() string and it will
+// lazily get Group from request info.
+type lazyAPIVersion struct {
+	req *http.Request
+}
+
+func (lazy *lazyAPIVersion) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.APIVersion
+		}
+	}
+
+	return "unknown"
+}
+
+// lazyName implements String() string and it will
+// lazily get Group from request info.
+type lazyName struct {
+	req *http.Request
+}
+
+func (lazy *lazyName) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.Name
+		}
+	}
+
+	return "unknown"
+}
+
+// lazySubresource implements String() string and it will
+// lazily get Group from request info.
+type lazySubresource struct {
+	req *http.Request
+}
+
+func (lazy *lazySubresource) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.Subresource
+		}
+	}
+
+	return "unknown"
+}
+
+// lazyNamespace implements String() string and it will
+// lazily get Group from request info.
+type lazyNamespace struct {
+	req *http.Request
+}
+
+func (lazy *lazyNamespace) String() string {
+	if lazy.req != nil {
+		ctx := lazy.req.Context()
+		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
+		if ok {
+			return requestInfo.Namespace
+		}
+	}
+
+	return "unknown"
+}
+
 // lazyAuditID implements Stringer interface to lazily retrieve
 // the audit ID associated with the request.
 type lazyAuditID struct {

vendor/k8s.io/apiserver/pkg/endpoints/handlers/metrics/metrics.go

@@ -18,7 +18,10 @@ package metrics
 
 import (
 	"context"
+	"sync"
+
 	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 type RequestBodyVerb string
@@ -35,8 +38,8 @@ var (
 	RequestBodySizes = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
 			Subsystem: "apiserver",
-			Name:      "request_body_sizes",
-			Help:      "Apiserver request body sizes broken out by size.",
+			Name:      "request_body_size_bytes",
+			Help:      "Apiserver request body size in bytes broken out by resource and verb.",
 			// we use 0.05 KB as the smallest bucket with 0.1 KB increments up to the
 			// apiserver limit.
 			Buckets:        metrics.LinearBuckets(50000, 100000, 31),
@@ -46,6 +49,15 @@ var (
 	)
 )
 
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(RequestBodySizes)
+	})
+}
+
 func RecordRequestBodySize(ctx context.Context, resource string, verb RequestBodyVerb, size int) {
 	RequestBodySizes.WithContext(ctx).WithLabelValues(resource, string(verb)).Observe(float64(size))
 }

vendor/k8s.io/apiserver/pkg/endpoints/handlers/response.go

@@ -18,8 +18,11 @@ package handlers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
+	"reflect"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -29,48 +32,228 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1beta1/validation"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
 	endpointsrequest "k8s.io/apiserver/pkg/endpoints/request"
+
+	klog "k8s.io/klog/v2"
 )
 
-// transformObject takes the object as returned by storage and ensures it is in
-// the client's desired form, as well as ensuring any API level fields like self-link
-// are properly set.
-func transformObject(ctx context.Context, obj runtime.Object, opts interface{}, mediaType negotiation.MediaTypeOptions, scope *RequestScope, req *http.Request) (runtime.Object, error) {
-	if co, ok := obj.(runtime.CacheableObject); ok {
-		if mediaType.Convert != nil {
-			// Non-nil mediaType.Convert means that some conversion of the object
-			// has to happen. Currently conversion may potentially modify the
-			// object or assume something about it (e.g. asTable operates on
-			// reflection, which won't work for any wrapper).
-			// To ensure it will work correctly, let's operate on base objects
-			// and not cache it for now.
-			//
-			// TODO: Long-term, transformObject should be changed so that it
-			// implements runtime.Encoder interface.
-			return doTransformObject(ctx, co.GetObject(), opts, mediaType, scope, req)
-		}
-	}
-	return doTransformObject(ctx, obj, opts, mediaType, scope, req)
+// watchEmbeddedEncoder performs encoding of the embedded object.
+//
+// NOTE: watchEmbeddedEncoder is NOT thread-safe.
+type watchEmbeddedEncoder struct {
+	encoder runtime.Encoder
+
+	ctx context.Context
+
+	// target, if non-nil, configures transformation type.
+	// The other options are ignored if target is nil.
+	target       *schema.GroupVersionKind
+	tableOptions *metav1.TableOptions
+	scope        *RequestScope
+
+	// identifier of the encoder, computed lazily
+	identifier runtime.Identifier
 }
 
-func doTransformObject(ctx context.Context, obj runtime.Object, opts interface{}, mediaType negotiation.MediaTypeOptions, scope *RequestScope, req *http.Request) (runtime.Object, error) {
+func newWatchEmbeddedEncoder(ctx context.Context, encoder runtime.Encoder, target *schema.GroupVersionKind, tableOptions *metav1.TableOptions, scope *RequestScope) *watchEmbeddedEncoder {
+	return &watchEmbeddedEncoder{
+		encoder:      encoder,
+		ctx:          ctx,
+		target:       target,
+		tableOptions: tableOptions,
+		scope:        scope,
+	}
+}
+
+// Encode implements runtime.Encoder interface.
+func (e *watchEmbeddedEncoder) Encode(obj runtime.Object, w io.Writer) error {
+	if co, ok := obj.(runtime.CacheableObject); ok {
+		return co.CacheEncode(e.Identifier(), e.doEncode, w)
+	}
+	return e.doEncode(obj, w)
+}
+
+func (e *watchEmbeddedEncoder) doEncode(obj runtime.Object, w io.Writer) error {
+	result, err := doTransformObject(e.ctx, obj, e.tableOptions, e.target, e.scope)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failed to transform object %v: %v", reflect.TypeOf(obj), err))
+		result = obj
+	}
+
+	// When we are tranforming to a table, use the original table options when
+	// we should print headers only on the first object - headers should be
+	// omitted on subsequent events.
+	if e.tableOptions != nil && !e.tableOptions.NoHeaders {
+		e.tableOptions.NoHeaders = true
+		// With options change, we should recompute the identifier.
+		// Clearing this will trigger lazy recompute when needed.
+		e.identifier = ""
+	}
+
+	return e.encoder.Encode(result, w)
+}
+
+// Identifier implements runtime.Encoder interface.
+func (e *watchEmbeddedEncoder) Identifier() runtime.Identifier {
+	if e.identifier == "" {
+		e.identifier = e.embeddedIdentifier()
+	}
+	return e.identifier
+}
+
+type watchEmbeddedEncoderIdentifier struct {
+	Name      string              `json:"name,omitempty"`
+	Encoder   string              `json:"encoder,omitempty"`
+	Target    string              `json:"target,omitempty"`
+	Options   metav1.TableOptions `json:"options,omitempty"`
+	NoHeaders bool                `json:"noHeaders,omitempty"`
+}
+
+func (e *watchEmbeddedEncoder) embeddedIdentifier() runtime.Identifier {
+	if e.target == nil {
+		// If no conversion is performed, we effective only use
+		// the embedded identifier.
+		return e.encoder.Identifier()
+	}
+	identifier := watchEmbeddedEncoderIdentifier{
+		Name:    "watch-embedded",
+		Encoder: string(e.encoder.Identifier()),
+		Target:  e.target.String(),
+	}
+	if e.target.Kind == "Table" && e.tableOptions != nil {
+		identifier.Options = *e.tableOptions
+		identifier.NoHeaders = e.tableOptions.NoHeaders
+	}
+
+	result, err := json.Marshal(identifier)
+	if err != nil {
+		klog.Fatalf("Failed marshaling identifier for watchEmbeddedEncoder: %v", err)
+	}
+	return runtime.Identifier(result)
+}
+
+// watchEncoder performs encoding of the watch events.
+//
+// NOTE: watchEncoder is NOT thread-safe.
+type watchEncoder struct {
+	ctx             context.Context
+	kind            schema.GroupVersionKind
+	embeddedEncoder runtime.Encoder
+	encoder         runtime.Encoder
+	framer          io.Writer
+
+	buffer      runtime.Splice
+	eventBuffer runtime.Splice
+
+	currentEmbeddedIdentifier runtime.Identifier
+	identifiers               map[watch.EventType]runtime.Identifier
+}
+
+func newWatchEncoder(ctx context.Context, kind schema.GroupVersionKind, embeddedEncoder runtime.Encoder, encoder runtime.Encoder, framer io.Writer) *watchEncoder {
+	return &watchEncoder{
+		ctx:             ctx,
+		kind:            kind,
+		embeddedEncoder: embeddedEncoder,
+		encoder:         encoder,
+		framer:          framer,
+		buffer:          runtime.NewSpliceBuffer(),
+		eventBuffer:     runtime.NewSpliceBuffer(),
+	}
+}
+
+// Encode encodes a given watch event.
+// NOTE: if events object is implementing the CacheableObject interface,
+//
+//	the serialized version is cached in that object [not the event itself].
+func (e *watchEncoder) Encode(event watch.Event) error {
+	encodeFunc := func(obj runtime.Object, w io.Writer) error {
+		return e.doEncode(obj, event, w)
+	}
+	if co, ok := event.Object.(runtime.CacheableObject); ok {
+		return co.CacheEncode(e.identifier(event.Type), encodeFunc, e.framer)
+	}
+	return encodeFunc(event.Object, e.framer)
+}
+
+func (e *watchEncoder) doEncode(obj runtime.Object, event watch.Event, w io.Writer) error {
+	defer e.buffer.Reset()
+
+	if err := e.embeddedEncoder.Encode(obj, e.buffer); err != nil {
+		return fmt.Errorf("unable to encode watch object %T: %v", obj, err)
+	}
+
+	// ContentType is not required here because we are defaulting to the serializer type.
+	outEvent := &metav1.WatchEvent{
+		Type:   string(event.Type),
+		Object: runtime.RawExtension{Raw: e.buffer.Bytes()},
+	}
+	metrics.WatchEventsSizes.WithContext(e.ctx).WithLabelValues(e.kind.Group, e.kind.Version, e.kind.Kind).Observe(float64(len(outEvent.Object.Raw)))
+
+	defer e.eventBuffer.Reset()
+	if err := e.encoder.Encode(outEvent, e.eventBuffer); err != nil {
+		return fmt.Errorf("unable to encode watch object %T: %v (%#v)", outEvent, err, e)
+	}
+
+	_, err := w.Write(e.eventBuffer.Bytes())
+	return err
+}
+
+type watchEncoderIdentifier struct {
+	Name            string `json:"name,omitempty"`
+	EmbeddedEncoder string `json:"embeddedEncoder,omitempty"`
+	Encoder         string `json:"encoder,omitempty"`
+	EventType       string `json:"eventType,omitempty"`
+}
+
+func (e *watchEncoder) identifier(eventType watch.EventType) runtime.Identifier {
+	// We need to take into account that in embeddedEncoder includes table
+	// transformer, then its identifier is dynamic. As a result, whenever
+	// the identifier of embeddedEncoder changes, we need to invalidate the
+	// whole identifiers cache.
+	// TODO(wojtek-t): Can we optimize it somehow?
+	if e.currentEmbeddedIdentifier != e.embeddedEncoder.Identifier() {
+		e.currentEmbeddedIdentifier = e.embeddedEncoder.Identifier()
+		e.identifiers = map[watch.EventType]runtime.Identifier{}
+	}
+	if _, ok := e.identifiers[eventType]; !ok {
+		e.identifiers[eventType] = e.typeIdentifier(eventType)
+	}
+	return e.identifiers[eventType]
+}
+
+func (e *watchEncoder) typeIdentifier(eventType watch.EventType) runtime.Identifier {
+	// The eventType is a non-standard pattern. This is coming from the fact
+	// that we're effectively serializing the whole watch event, but storing
+	// it in serializations of the Object within the watch event.
+	identifier := watchEncoderIdentifier{
+		Name:            "watch",
+		EmbeddedEncoder: string(e.embeddedEncoder.Identifier()),
+		Encoder:         string(e.encoder.Identifier()),
+		EventType:       string(eventType),
+	}
+
+	result, err := json.Marshal(identifier)
+	if err != nil {
+		klog.Fatalf("Failed marshaling identifier for watchEncoder: %v", err)
+	}
+	return runtime.Identifier(result)
+}
+
+// doTransformResponseObject is used for handling all requests, including watch.
+func doTransformObject(ctx context.Context, obj runtime.Object, opts interface{}, target *schema.GroupVersionKind, scope *RequestScope) (runtime.Object, error) {
 	if _, ok := obj.(*metav1.Status); ok {
 		return obj, nil
 	}
 
-	// ensure that for empty lists we don't return <nil> items.
-	// This is safe to modify without deep-copying the object, as
-	// List objects themselves are never cached.
-	if meta.IsListType(obj) && meta.LenList(obj) == 0 {
-		if err := meta.SetList(obj, []runtime.Object{}); err != nil {
-			return nil, err
-		}
-	}
-
-	switch target := mediaType.Convert; {
+	switch {
 	case target == nil:
+		// If we ever change that from a no-op, the identifier of
+		// the watchEmbeddedEncoder has to be adjusted accordingly.
 		return obj, nil
 
 	case target.Kind == "PartialObjectMetadata":
@@ -128,6 +311,7 @@ func targetEncodingForTransform(scope *RequestScope, mediaType negotiation.Media
 
 // transformResponseObject takes an object loaded from storage and performs any necessary transformations.
 // Will write the complete response object.
+// transformResponseObject is used only for handling non-streaming requests.
 func transformResponseObject(ctx context.Context, scope *RequestScope, req *http.Request, w http.ResponseWriter, statusCode int, mediaType negotiation.MediaTypeOptions, result runtime.Object) {
 	options, err := optionsForTransform(mediaType, req)
 	if err != nil {
@@ -135,9 +319,19 @@ func transformResponseObject(ctx context.Context, scope *RequestScope, req *http
 		return
 	}
 
+	// ensure that for empty lists we don't return <nil> items.
+	// This is safe to modify without deep-copying the object, as
+	// List objects themselves are never cached.
+	if meta.IsListType(result) && meta.LenList(result) == 0 {
+		if err := meta.SetList(result, []runtime.Object{}); err != nil {
+			scope.err(err, w, req)
+			return
+		}
+	}
+
 	var obj runtime.Object
 	do := func() {
-		obj, err = transformObject(ctx, result, options, mediaType, scope, req)
+		obj, err = doTransformObject(ctx, result, options, mediaType.Convert, scope)
 	}
 	endpointsrequest.TrackTransformResponseObjectLatency(ctx, do)
 

vendor/k8s.io/apiserver/pkg/endpoints/handlers/trace_util.go

@@ -27,6 +27,11 @@ func traceFields(req *http.Request) []attribute.KeyValue {
 		attribute.Stringer("accept", &lazyAccept{req: req}),
 		attribute.Stringer("audit-id", &lazyAuditID{req: req}),
 		attribute.Stringer("client", &lazyClientIP{req: req}),
+		attribute.Stringer("api-group", &lazyAPIGroup{req: req}),
+		attribute.Stringer("api-version", &lazyAPIVersion{req: req}),
+		attribute.Stringer("name", &lazyName{req: req}),
+		attribute.Stringer("subresource", &lazySubresource{req: req}),
+		attribute.Stringer("namespace", &lazyNamespace{req: req}),
 		attribute.String("protocol", req.Proto),
 		attribute.Stringer("resource", &lazyResource{req: req}),
 		attribute.Stringer("scope", &lazyScope{req: req}),

vendor/k8s.io/apiserver/pkg/endpoints/handlers/watch.go

@@ -19,9 +19,7 @@ package handlers
 import (
 	"bytes"
 	"fmt"
-	"io"
 	"net/http"
-	"reflect"
 	"time"
 
 	"golang.org/x/net/websocket"
@@ -29,13 +27,15 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/serializer/streaming"
 	"k8s.io/apimachinery/pkg/util/httpstream/wsstream"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/features"
+	"k8s.io/apiserver/pkg/storage"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 )
 
 // nothing will ever be sent down this channel
@@ -63,7 +63,7 @@ func (w *realTimeoutFactory) TimeoutCh() (<-chan time.Time, func() bool) {
 
 // serveWatch will serve a watch response.
 // TODO: the functionality in this method and in WatchServer.Serve is not cleanly decoupled.
-func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions negotiation.MediaTypeOptions, req *http.Request, w http.ResponseWriter, timeout time.Duration) {
+func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions negotiation.MediaTypeOptions, req *http.Request, w http.ResponseWriter, timeout time.Duration, metricsScope string) {
 	defer watcher.Stop()
 
 	options, err := optionsForTransform(mediaTypeOptions, req)
@@ -92,6 +92,8 @@ func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions n
 		mediaType += ";stream=watch"
 	}
 
+	ctx := req.Context()
+
 	// locate the appropriate embedded encoder based on the transform
 	var embeddedEncoder runtime.Encoder
 	contentKind, contentSerializer, transform := targetEncodingForTransform(scope, mediaTypeOptions, req)
@@ -106,13 +108,41 @@ func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions n
 		embeddedEncoder = scope.Serializer.EncoderForVersion(serializer.Serializer, contentKind.GroupVersion())
 	}
 
+	var memoryAllocator runtime.MemoryAllocator
+
+	if encoderWithAllocator, supportsAllocator := embeddedEncoder.(runtime.EncoderWithAllocator); supportsAllocator {
+		// don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
+		// instead, we allocate the buffer for the entire watch session and release it when we close the connection.
+		memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
+		defer runtime.AllocatorPool.Put(memoryAllocator)
+		embeddedEncoder = runtime.NewEncoderWithAllocator(encoderWithAllocator, memoryAllocator)
+	}
+	var tableOptions *metav1.TableOptions
+	if options != nil {
+		if passedOptions, ok := options.(*metav1.TableOptions); ok {
+			tableOptions = passedOptions
+		} else {
+			scope.err(fmt.Errorf("unexpected options type: %T", options), w, req)
+			return
+		}
+	}
+	embeddedEncoder = newWatchEmbeddedEncoder(ctx, embeddedEncoder, mediaTypeOptions.Convert, tableOptions, scope)
+
+	if encoderWithAllocator, supportsAllocator := encoder.(runtime.EncoderWithAllocator); supportsAllocator {
+		if memoryAllocator == nil {
+			// don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
+			// instead, we allocate the buffer for the entire watch session and release it when we close the connection.
+			memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
+			defer runtime.AllocatorPool.Put(memoryAllocator)
+		}
+		encoder = runtime.NewEncoderWithAllocator(encoderWithAllocator, memoryAllocator)
+	}
+
 	var serverShuttingDownCh <-chan struct{}
 	if signals := apirequest.ServerShutdownSignalFrom(req.Context()); signals != nil {
 		serverShuttingDownCh = signals.ShuttingDown()
 	}
 
-	ctx := req.Context()
-
 	server := &WatchServer{
 		Watching: watcher,
 		Scope:    scope,
@@ -123,23 +153,10 @@ func serveWatch(watcher watch.Interface, scope *RequestScope, mediaTypeOptions n
 		Encoder:         encoder,
 		EmbeddedEncoder: embeddedEncoder,
 
-		Fixup: func(obj runtime.Object) runtime.Object {
-			result, err := transformObject(ctx, obj, options, mediaTypeOptions, scope, req)
-			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("failed to transform object %v: %v", reflect.TypeOf(obj), err))
-				return obj
-			}
-			// When we are transformed to a table, use the table options as the state for whether we
-			// should print headers - on watch, we only want to print table headers on the first object
-			// and omit them on subsequent events.
-			if tableOptions, ok := options.(*metav1.TableOptions); ok {
-				tableOptions.NoHeaders = true
-			}
-			return result
-		},
-
 		TimeoutFactory:       &realTimeoutFactory{timeout},
 		ServerShuttingDownCh: serverShuttingDownCh,
+
+		metricsScope: metricsScope,
 	}
 
 	server.ServeHTTP(w, req)
@@ -160,11 +177,11 @@ type WatchServer struct {
 	Encoder runtime.Encoder
 	// used to encode the nested object in the watch stream
 	EmbeddedEncoder runtime.Encoder
-	// used to correct the object before we send it to the serializer
-	Fixup func(runtime.Object) runtime.Object
 
 	TimeoutFactory       TimeoutFactory
 	ServerShuttingDownCh <-chan struct{}
+
+	metricsScope string
 }
 
 // ServeHTTP serves a series of encoded events via HTTP with Transfer-Encoding: chunked
@@ -195,17 +212,6 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	var e streaming.Encoder
-	var memoryAllocator runtime.MemoryAllocator
-
-	if encoder, supportsAllocator := s.Encoder.(runtime.EncoderWithAllocator); supportsAllocator {
-		memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
-		defer runtime.AllocatorPool.Put(memoryAllocator)
-		e = streaming.NewEncoderWithAllocator(framer, encoder, memoryAllocator)
-	} else {
-		e = streaming.NewEncoder(framer, s.Encoder)
-	}
-
 	// ensure the connection times out
 	timeoutCh, cleanup := s.TimeoutFactory.TimeoutCh()
 	defer cleanup()
@@ -216,26 +222,10 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(http.StatusOK)
 	flusher.Flush()
 
-	var unknown runtime.Unknown
-	internalEvent := &metav1.InternalEvent{}
-	outEvent := &metav1.WatchEvent{}
-	buf := runtime.NewSpliceBuffer()
+	watchEncoder := newWatchEncoder(req.Context(), kind, s.EmbeddedEncoder, s.Encoder, framer)
 	ch := s.Watching.ResultChan()
 	done := req.Context().Done()
 
-	embeddedEncodeFn := s.EmbeddedEncoder.Encode
-	if encoder, supportsAllocator := s.EmbeddedEncoder.(runtime.EncoderWithAllocator); supportsAllocator {
-		if memoryAllocator == nil {
-			// don't put the allocator inside the embeddedEncodeFn as that would allocate memory on every call.
-			// instead, we allocate the buffer for the entire watch session and release it when we close the connection.
-			memoryAllocator = runtime.AllocatorPool.Get().(*runtime.Allocator)
-			defer runtime.AllocatorPool.Put(memoryAllocator)
-		}
-		embeddedEncodeFn = func(obj runtime.Object, w io.Writer) error {
-			return encoder.EncodeWithAllocator(obj, w, memoryAllocator)
-		}
-	}
-
 	for {
 		select {
 		case <-s.ServerShuttingDownCh:
@@ -257,42 +247,20 @@ func (s *WatchServer) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 				return
 			}
 			metrics.WatchEvents.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Inc()
+			isWatchListLatencyRecordingRequired := shouldRecordWatchListLatency(event)
 
-			obj := s.Fixup(event.Object)
-			if err := embeddedEncodeFn(obj, buf); err != nil {
-				// unexpected error
-				utilruntime.HandleError(fmt.Errorf("unable to encode watch object %T: %v", obj, err))
-				return
-			}
-
-			// ContentType is not required here because we are defaulting to the serializer
-			// type
-			unknown.Raw = buf.Bytes()
-			event.Object = &unknown
-			metrics.WatchEventsSizes.WithContext(req.Context()).WithLabelValues(kind.Group, kind.Version, kind.Kind).Observe(float64(len(unknown.Raw)))
-
-			*outEvent = metav1.WatchEvent{}
-
-			// create the external type directly and encode it.  Clients will only recognize the serialization we provide.
-			// The internal event is being reused, not reallocated so its just a few extra assignments to do it this way
-			// and we get the benefit of using conversion functions which already have to stay in sync
-			*internalEvent = metav1.InternalEvent(event)
-			err := metav1.Convert_v1_InternalEvent_To_v1_WatchEvent(internalEvent, outEvent, nil)
-			if err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to convert watch object: %v", err))
-				// client disconnect.
-				return
-			}
-			if err := e.Encode(outEvent); err != nil {
-				utilruntime.HandleError(fmt.Errorf("unable to encode watch object %T: %v (%#v)", outEvent, err, e))
+			if err := watchEncoder.Encode(event); err != nil {
+				utilruntime.HandleError(err)
 				// client disconnect.
 				return
 			}
+
 			if len(ch) == 0 {
 				flusher.Flush()
 			}
-
-			buf.Reset()
+			if isWatchListLatencyRecordingRequired {
+				metrics.RecordWatchListLatency(req.Context(), s.Scope.Resource, s.metricsScope)
+			}
 		}
 	}
 }
@@ -326,10 +294,10 @@ func (s *WatchServer) HandleWS(ws *websocket.Conn) {
 				// End of results.
 				return
 			}
-			obj := s.Fixup(event.Object)
-			if err := s.EmbeddedEncoder.Encode(obj, buf); err != nil {
+
+			if err := s.EmbeddedEncoder.Encode(event.Object, buf); err != nil {
 				// unexpected error
-				utilruntime.HandleError(fmt.Errorf("unable to encode watch object %T: %v", obj, err))
+				utilruntime.HandleError(fmt.Errorf("unable to encode watch object %T: %v", event.Object, err))
 				return
 			}
 
@@ -371,3 +339,19 @@ func (s *WatchServer) HandleWS(ws *websocket.Conn) {
 		}
 	}
 }
+
+func shouldRecordWatchListLatency(event watch.Event) bool {
+	if event.Type != watch.Bookmark || !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
+		return false
+	}
+	// as of today the initial-events-end annotation is added only to a single event
+	// by the watch cache and only when certain conditions are met
+	//
+	// for more please read https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/3157-watch-list
+	hasAnnotation, err := storage.HasInitialEventsEndBookmarkAnnotation(event.Object)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("unable to determine if the obj has the required annotation for measuring watchlist latency, obj %T: %v", event.Object, err))
+		return false
+	}
+	return hasAnnotation
+}

vendor/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -796,7 +796,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			}
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("read"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Returns(http.StatusOK, "OK", producedObject).
@@ -817,7 +817,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("list"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), allMediaTypes...)...).
 				Returns(http.StatusOK, "OK", versionedList).
@@ -850,7 +850,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.PUT(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("replace"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Returns(http.StatusOK, "OK", producedObject).
@@ -879,7 +879,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.PATCH(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Consumes(supportedTypes...).
 				Operation("patch"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
@@ -909,7 +909,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			}
 			route := ws.POST(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("create"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Returns(http.StatusOK, "OK", producedObject).
@@ -938,7 +938,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.DELETE(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("delete"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Writes(deleteReturnType).
@@ -962,7 +962,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.DELETE(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("deletecollection"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(append(storageMeta.ProducesMIMETypes(action.Verb), mediaTypes...)...).
 				Writes(versionedStatus).
@@ -990,7 +990,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("watch"+namespaced+kind+strings.Title(subresource)+operationSuffix).
 				Produces(allMediaTypes...).
 				Returns(http.StatusOK, "OK", versionedWatchEvent).
@@ -1011,7 +1011,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			handler = utilwarning.AddWarningsHandler(handler, warnings)
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
-				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
+				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed. Defaults to 'false' unless the user-agent indicates a browser or command-line HTTP tool (curl and wget).")).
 				Operation("watch"+namespaced+kind+strings.Title(subresource)+"List"+operationSuffix).
 				Produces(allMediaTypes...).
 				Returns(http.StatusOK, "OK", versionedWatchEvent).

vendor/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -18,6 +18,7 @@ package metrics
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -26,8 +27,12 @@ import (
 	"time"
 
 	restful "github.com/emicklei/go-restful/v3"
+
+	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	utilsets "k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authentication/user"
@@ -280,6 +285,17 @@ var (
 		[]string{"code_path"},
 	)
 
+	watchListLatencies = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Subsystem:      APIServerComponent,
+			Name:           "watch_list_duration_seconds",
+			Help:           "Response latency distribution in seconds for watch list requests broken by group, version, resource and scope.",
+			Buckets:        []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 2, 4, 6, 8, 10, 15, 20, 30, 45, 60},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"group", "version", "resource", "scope"},
+	)
+
 	metrics = []resettableCollector{
 		deprecatedRequestGauge,
 		requestCounter,
@@ -300,6 +316,7 @@ var (
 		requestAbortsTotal,
 		requestPostTimeoutTotal,
 		requestTimestampComparisonDuration,
+		watchListLatencies,
 	}
 
 	// these are the valid request methods which we report in our metrics. Any other request methods
@@ -511,6 +528,18 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
 	fn()
 }
 
+// RecordWatchListLatency simply records response latency for watch list requests.
+func RecordWatchListLatency(ctx context.Context, gvr schema.GroupVersionResource, metricsScope string) {
+	requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
+	if !ok {
+		utilruntime.HandleError(fmt.Errorf("unable to measure watchlist latency because no received ts found in the ctx, gvr: %s", gvr))
+		return
+	}
+	elapsedSeconds := time.Since(requestReceivedTimestamp).Seconds()
+
+	watchListLatencies.WithContext(ctx).WithLabelValues(gvr.Group, gvr.Version, gvr.Resource, metricsScope).Observe(elapsedSeconds)
+}
+
 // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
 // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
 func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, httpCode, respSize int, elapsed time.Duration) {
@@ -621,6 +650,26 @@ func CleanScope(requestInfo *request.RequestInfo) string {
 	return ""
 }
 
+// CleanListScope computes the request scope for metrics.
+//
+// Note that normally we would use CleanScope for computation.
+// But due to the same reasons mentioned in determineRequestNamespaceAndName we cannot.
+func CleanListScope(ctx context.Context, opts *metainternalversion.ListOptions) string {
+	namespace, name := determineRequestNamespaceAndName(ctx, opts)
+	if len(name) > 0 {
+		return "resource"
+	}
+	if len(namespace) > 0 {
+		return "namespace"
+	}
+	if requestInfo, ok := request.RequestInfoFrom(ctx); ok {
+		if requestInfo.IsResourceRequest {
+			return "cluster"
+		}
+	}
+	return ""
+}
+
 // CanonicalVerb distinguishes LISTs from GETs (and HEADs). It assumes verb is
 // UPPERCASE.
 func CanonicalVerb(verb string, scope string) string {
@@ -655,6 +704,30 @@ func CleanVerb(verb string, request *http.Request, requestInfo *request.RequestI
 	return reportedVerb
 }
 
+// determineRequestNamespaceAndName computes name and namespace for the given requests
+//
+// note that the logic of this function was copy&pasted from cacher.go
+// after an unsuccessful attempt of moving it to RequestInfo
+//
+// see: https://github.com/kubernetes/kubernetes/pull/120520
+func determineRequestNamespaceAndName(ctx context.Context, opts *metainternalversion.ListOptions) (namespace, name string) {
+	if requestNamespace, ok := request.NamespaceFrom(ctx); ok && len(requestNamespace) > 0 {
+		namespace = requestNamespace
+	} else if opts != nil && opts.FieldSelector != nil {
+		if selectorNamespace, ok := opts.FieldSelector.RequiresExactMatch("metadata.namespace"); ok {
+			namespace = selectorNamespace
+		}
+	}
+	if requestInfo, ok := request.RequestInfoFrom(ctx); ok && requestInfo != nil && len(requestInfo.Name) > 0 {
+		name = requestInfo.Name
+	} else if opts != nil && opts.FieldSelector != nil {
+		if selectorName, ok := opts.FieldSelector.RequiresExactMatch("metadata.name"); ok {
+			name = selectorName
+		}
+	}
+	return
+}
+
 // cleanVerb additionally ensures that unknown verbs don't clog up the metrics.
 func cleanVerb(verb, suggestedVerb string, request *http.Request, requestInfo *request.RequestInfo) string {
 	// CanonicalVerb (being an input for this function) doesn't handle correctly the

vendor/k8s.io/apiserver/pkg/features/kube_features.go

@@ -54,6 +54,7 @@ const (
 	// owner: @smarterclayton
 	// alpha: v1.8
 	// beta: v1.9
+	// stable: 1.29
 	//
 	// Allow API clients to retrieve resource lists in chunks rather than
 	// all at once.
@@ -62,6 +63,7 @@ const (
 	// owner: @MikeSpreitzer @yue9944882
 	// alpha: v1.18
 	// beta: v1.20
+	// stable: 1.29
 	//
 	// Enables managing request concurrency with prioritization and fairness at each server.
 	// The FeatureGate was introduced in release 1.15 but the feature
@@ -99,6 +101,7 @@ const (
 	// kep: https://kep.k8s.io/2876
 	// alpha: v1.23
 	// beta: v1.25
+	// stable: v1.29
 	//
 	// Enables expression validation for Custom Resource
 	CustomResourceValidationExpressions featuregate.Feature = "CustomResourceValidationExpressions"
@@ -121,6 +124,7 @@ const (
 	// kep: https://kep.k8s.io/3299
 	// alpha: v1.25
 	// beta: v1.27
+	// stable: v1.29
 	//
 	// Enables KMS v2 API for encryption at rest.
 	KMSv2 featuregate.Feature = "KMSv2"
@@ -128,6 +132,7 @@ const (
 	// owner: @enj
 	// kep: https://kep.k8s.io/3299
 	// beta: v1.28
+	// stable: v1.29
 	//
 	// Enables the use of derived encryption keys with KMS v2.
 	KMSv2KDF featuregate.Feature = "KMSv2KDF"
@@ -141,18 +146,10 @@ const (
 	// in the spec returned from kube-apiserver.
 	OpenAPIEnums featuregate.Feature = "OpenAPIEnums"
 
-	// owner: @jefftree
-	// kep: https://kep.k8s.io/2896
-	// alpha: v1.23
-	// beta: v1.24
-	// stable: v1.27
-	//
-	// Enables kubernetes to publish OpenAPI v3
-	OpenAPIV3 featuregate.Feature = "OpenAPIV3"
-
 	// owner: @caesarxuchao
 	// alpha: v1.15
 	// beta: v1.16
+	// stable: 1.29
 	//
 	// Allow apiservers to show a count of remaining items in the response
 	// to a chunking list request.
@@ -214,6 +211,20 @@ const (
 	// document.
 	StorageVersionHash featuregate.Feature = "StorageVersionHash"
 
+	// owner: @aramase, @enj, @nabokihms
+	// kep: https://kep.k8s.io/3331
+	// alpha: v1.29
+	//
+	// Enables Structured Authentication Configuration
+	StructuredAuthenticationConfiguration featuregate.Feature = "StructuredAuthenticationConfiguration"
+
+	// owner: @palnabarun
+	// kep: https://kep.k8s.io/3221
+	// alpha: v1.29
+	//
+	// Enables Structured Authorization Configuration
+	StructuredAuthorizationConfiguration featuregate.Feature = "StructuredAuthorizationConfiguration"
+
 	// owner: @wojtek-t
 	// alpha: v1.15
 	// beta: v1.16
@@ -241,6 +252,14 @@ const (
 	//
 	// Allow the API server to serve consistent lists from cache
 	ConsistentListFromCache featuregate.Feature = "ConsistentListFromCache"
+
+	// owner: @tkashem
+	// beta: v1.29
+	//
+	// Allow Priority & Fairness in the API server to use a zero value for
+	// the 'nominalConcurrencyShares' field of the 'limited' section of a
+	// priority level.
+	ZeroLimitedNominalConcurrencyShares featuregate.Feature = "ZeroLimitedNominalConcurrencyShares"
 )
 
 func init() {
@@ -256,9 +275,9 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	AdmissionWebhookMatchConditions: {Default: true, PreRelease: featuregate.Beta},
 
-	APIListChunking: {Default: true, PreRelease: featuregate.Beta},
+	APIListChunking: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32
 
-	APIPriorityAndFairness: {Default: true, PreRelease: featuregate.Beta},
+	APIPriorityAndFairness: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
 	APIResponseCompression: {Default: true, PreRelease: featuregate.Beta},
 
@@ -268,21 +287,19 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Beta},
 
-	CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.Beta},
+	CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
 	EfficientWatchResumption: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
-	KMSv1: {Default: true, PreRelease: featuregate.Deprecated},
+	KMSv1: {Default: false, PreRelease: featuregate.Deprecated},
 
-	KMSv2: {Default: true, PreRelease: featuregate.Beta},
+	KMSv2: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
-	KMSv2KDF: {Default: false, PreRelease: featuregate.Beta}, // default and lock to true in 1.29, remove in 1.31
+	KMSv2KDF: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31
 
 	OpenAPIEnums: {Default: true, PreRelease: featuregate.Beta},
 
-	OpenAPIV3: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.29
-
-	RemainingItemCount: {Default: true, PreRelease: featuregate.Beta},
+	RemainingItemCount: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.32
 
 	RemoveSelfLink: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
@@ -294,7 +311,11 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	StorageVersionHash: {Default: true, PreRelease: featuregate.Beta},
 
-	UnauthenticatedHTTP2DOSMitigation: {Default: false, PreRelease: featuregate.Beta},
+	StructuredAuthenticationConfiguration: {Default: false, PreRelease: featuregate.Alpha},
+
+	StructuredAuthorizationConfiguration: {Default: false, PreRelease: featuregate.Alpha},
+
+	UnauthenticatedHTTP2DOSMitigation: {Default: true, PreRelease: featuregate.Beta},
 
 	WatchBookmark: {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 
@@ -303,4 +324,6 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	WatchList: {Default: false, PreRelease: featuregate.Alpha},
 
 	ConsistentListFromCache: {Default: false, PreRelease: featuregate.Alpha},
+
+	ZeroLimitedNominalConcurrencyShares: {Default: false, PreRelease: featuregate.Beta},
 }

vendor/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go

@@ -44,7 +44,7 @@ func StorageWithCacher() generic.StorageDecorator {
 		triggerFuncs storage.IndexerFuncs,
 		indexers *cache.Indexers) (storage.Interface, factory.DestroyFunc, error) {
 
-		s, d, err := generic.NewRawStorage(storageConfig, newFunc)
+		s, d, err := generic.NewRawStorage(storageConfig, newFunc, newListFunc, resourcePrefix)
 		if err != nil {
 			return s, d, err
 		}

vendor/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go

@@ -47,12 +47,12 @@ func UndecoratedStorage(
 	getAttrsFunc storage.AttrFunc,
 	trigger storage.IndexerFuncs,
 	indexers *cache.Indexers) (storage.Interface, factory.DestroyFunc, error) {
-	return NewRawStorage(config, newFunc)
+	return NewRawStorage(config, newFunc, newListFunc, resourcePrefix)
 }
 
 // NewRawStorage creates the low level kv storage. This is a work-around for current
 // two layer of same storage interface.
 // TODO: Once cacher is enabled on all registries (event registry is special), we will remove this method.
-func NewRawStorage(config *storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, factory.DestroyFunc, error) {
-	return factory.Create(*config, newFunc)
+func NewRawStorage(config *storagebackend.ConfigForResource, newFunc, newListFunc func() runtime.Object, resourcePrefix string) (storage.Interface, factory.DestroyFunc, error) {
+	return factory.Create(*config, newFunc, newListFunc, resourcePrefix)
 }

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -78,6 +78,7 @@ import (
 	"k8s.io/component-base/tracing"
 	"k8s.io/klog/v2"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
+	"k8s.io/kube-openapi/pkg/spec3"
 	"k8s.io/kube-openapi/pkg/validation/spec"
 	"k8s.io/utils/clock"
 	utilsnet "k8s.io/utils/net"
@@ -194,7 +195,7 @@ type Config struct {
 	// OpenAPIConfig will be used in generating OpenAPI spec. This is nil by default. Use DefaultOpenAPIConfig for "working" defaults.
 	OpenAPIConfig *openapicommon.Config
 	// OpenAPIV3Config will be used in generating OpenAPI V3 spec. This is nil by default. Use DefaultOpenAPIV3Config for "working" defaults.
-	OpenAPIV3Config *openapicommon.Config
+	OpenAPIV3Config *openapicommon.OpenAPIV3Config
 	// SkipOpenAPIInstallation avoids installing the OpenAPI handler if set to true.
 	SkipOpenAPIInstallation bool
 
@@ -482,8 +483,23 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
 }
 
 // DefaultOpenAPIV3Config provides the default OpenAPIV3Config used to build the OpenAPI V3 spec
-func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.Config {
-	defaultConfig := DefaultOpenAPIConfig(getDefinitions, defNamer)
+func DefaultOpenAPIV3Config(getDefinitions openapicommon.GetOpenAPIDefinitions, defNamer *apiopenapi.DefinitionNamer) *openapicommon.OpenAPIV3Config {
+	defaultConfig := &openapicommon.OpenAPIV3Config{
+		IgnorePrefixes: []string{},
+		Info: &spec.Info{
+			InfoProps: spec.InfoProps{
+				Title: "Generic API Server",
+			},
+		},
+		DefaultResponse: &spec3.Response{
+			ResponseProps: spec3.ResponseProps{
+				Description: "Default Response.",
+			},
+		},
+		GetOperationIDAndTags: apiopenapi.GetOperationIDAndTags,
+		GetDefinitionName:     defNamer.GetDefinitionName,
+		GetDefinitions:        getDefinitions,
+	}
 	defaultConfig.Definitions = getDefinitions(func(name string) spec.Ref {
 		defName, _ := defaultConfig.GetDefinitionName(name)
 		return spec.MustCreateRef("#/components/schemas/" + openapicommon.EscapeJsonPointer(defName))
@@ -608,6 +624,45 @@ func completeOpenAPI(config *openapicommon.Config, version *version.Info) {
 	}
 }
 
+func completeOpenAPIV3(config *openapicommon.OpenAPIV3Config, version *version.Info) {
+	if config == nil {
+		return
+	}
+	if config.SecuritySchemes != nil {
+		// Setup OpenAPI security: all APIs will have the same authentication for now.
+		config.DefaultSecurity = []map[string][]string{}
+		keys := []string{}
+		for k := range config.SecuritySchemes {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			config.DefaultSecurity = append(config.DefaultSecurity, map[string][]string{k: {}})
+		}
+		if config.CommonResponses == nil {
+			config.CommonResponses = map[int]*spec3.Response{}
+		}
+		if _, exists := config.CommonResponses[http.StatusUnauthorized]; !exists {
+			config.CommonResponses[http.StatusUnauthorized] = &spec3.Response{
+				ResponseProps: spec3.ResponseProps{
+					Description: "Unauthorized",
+				},
+			}
+		}
+	}
+	// make sure we populate info, and info.version, if not manually set
+	if config.Info == nil {
+		config.Info = &spec.Info{}
+	}
+	if config.Info.Version == "" {
+		if version != nil {
+			config.Info.Version = strings.Split(version.String(), "-")[0]
+		} else {
+			config.Info.Version = "unversioned"
+		}
+	}
+}
+
 // DrainedNotify returns a lifecycle signal of genericapiserver already drained while shutting down.
 func (c *Config) DrainedNotify() <-chan struct{} {
 	return c.lifecycleSignals.InFlightRequestsDrained.Signaled()
@@ -633,7 +688,7 @@ func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedCo
 	}
 
 	completeOpenAPI(c.OpenAPIConfig, c.Version)
-	completeOpenAPI(c.OpenAPIV3Config, c.Version)
+	completeOpenAPIV3(c.OpenAPIV3Config, c.Version)
 
 	if c.DiscoveryAddresses == nil {
 		c.DiscoveryAddresses = discovery.DefaultAddresses{DefaultAddress: c.ExternalAddress}
@@ -669,6 +724,12 @@ func (c *RecommendedConfig) Complete() CompletedConfig {
 	return c.Config.Complete(c.SharedInformerFactory)
 }
 
+var allowedMediaTypes = []string{
+	runtime.ContentTypeJSON,
+	runtime.ContentTypeYAML,
+	runtime.ContentTypeProtobuf,
+}
+
 // New creates a new server which logically combines the handling chain with the passed server.
 // name is used to differentiate for logging. The handler chain in particular can be difficult as it starts delegating.
 // delegationTarget may not be nil.
@@ -676,6 +737,18 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 	if c.Serializer == nil {
 		return nil, fmt.Errorf("Genericapiserver.New() called with config.Serializer == nil")
 	}
+	for _, info := range c.Serializer.SupportedMediaTypes() {
+		var ok bool
+		for _, mt := range allowedMediaTypes {
+			if info.MediaType == mt {
+				ok = true
+				break
+			}
+		}
+		if !ok {
+			return nil, fmt.Errorf("refusing to create new apiserver %q with support for media type %q (allowed media types are: %s)", name, info.MediaType, strings.Join(allowedMediaTypes, ", "))
+		}
+	}
 	if c.LoopbackClientConfig == nil {
 		return nil, fmt.Errorf("Genericapiserver.New() called with config.LoopbackClientConfig == nil")
 	}
@@ -915,7 +988,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 		requestWorkEstimator := flowcontrolrequest.NewWorkEstimator(
 			c.StorageObjectCountTracker.Get, c.FlowControl.GetInterestedWatchCount, workEstimatorCfg, c.FlowControl.GetMaxSeats)
 		handler = filterlatency.TrackCompleted(handler)
-		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator)
+		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl, requestWorkEstimator, c.RequestTimeout/4)
 		handler = filterlatency.TrackStarted(handler, c.TracerProvider, "priorityandfairness")
 	} else {
 		handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
@@ -994,14 +1067,10 @@ func installAPI(s *GenericAPIServer, c *Config) {
 	if c.EnableMetrics {
 		if c.EnableProfiling {
 			routes.MetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
-			if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
-				slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
-			}
+			slis.SLIMetricsWithReset{}.Install(s.Handler.NonGoRestfulMux)
 		} else {
 			routes.DefaultMetrics{}.Install(s.Handler.NonGoRestfulMux)
-			if utilfeature.DefaultFeatureGate.Enabled(features.ComponentSLIs) {
-				slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
-			}
+			slis.SLIMetrics{}.Install(s.Handler.NonGoRestfulMux)
 		}
 	}
 
@@ -1015,7 +1084,7 @@ func installAPI(s *GenericAPIServer, c *Config) {
 			s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
 		}
 	}
-	if c.FlowControl != nil && utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIPriorityAndFairness) {
+	if c.FlowControl != nil {
 		c.FlowControl.Install(s.Handler.NonGoRestfulMux)
 	}
 }

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -21,7 +21,7 @@ import (
 	"context"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"sync/atomic"
 	"time"
 
@@ -98,7 +98,7 @@ func (c *DynamicFileCAContent) AddListener(listener Listener) {
 
 // loadCABundle determines the next set of content for the file.
 func (c *DynamicFileCAContent) loadCABundle() error {
-	caBundle, err := ioutil.ReadFile(c.filename)
+	caBundle, err := os.ReadFile(c.filename)
 	if err != nil {
 		return err
 	}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"os"
 	"sync/atomic"
 	"time"
 
@@ -80,11 +80,11 @@ func (c *DynamicCertKeyPairContent) AddListener(listener Listener) {
 
 // loadCertKeyPair determines the next set of content for the file.
 func (c *DynamicCertKeyPairContent) loadCertKeyPair() error {
-	cert, err := ioutil.ReadFile(c.certFile)
+	cert, err := os.ReadFile(c.certFile)
 	if err != nil {
 		return err
 	}
-	key, err := ioutil.ReadFile(c.keyFile)
+	key, err := os.ReadFile(c.keyFile)
 	if err != nil {
 		return err
 	}

vendor/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -18,7 +18,7 @@ package egressselector
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -51,7 +51,7 @@ func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSe
 		return nil, nil
 	}
 	// a file was provided, so we just read it.
-	data, err := ioutil.ReadFile(configFilePath)
+	data, err := os.ReadFile(configFilePath)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
 	}

vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -22,10 +22,10 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -277,7 +277,7 @@ func getTLSConfig(t *apiserver.TLSConfig) (*tls.Config, error) {
 	}
 	certPool := x509.NewCertPool()
 	if caCert != "" {
-		certBytes, err := ioutil.ReadFile(caCert)
+		certBytes, err := os.ReadFile(caCert)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
 		}

vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -26,7 +26,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	flowcontrol "k8s.io/api/flowcontrol/v1beta3"
+	flowcontrol "k8s.io/api/flowcontrol/v1"
 	apitypes "k8s.io/apimachinery/pkg/types"
 	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -35,6 +35,7 @@ import (
 	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
 	flowcontrolrequest "k8s.io/apiserver/pkg/util/flowcontrol/request"
 	"k8s.io/klog/v2"
+	utilsclock "k8s.io/utils/clock"
 )
 
 // PriorityAndFairnessClassification identifies the results of
@@ -78,6 +79,10 @@ type priorityAndFairnessHandler struct {
 	// the purpose of computing RetryAfter header to avoid system
 	// overload.
 	droppedRequests utilflowcontrol.DroppedRequestsTracker
+
+	// newReqWaitCtxFn creates a derived context with a deadline
+	// of how long a given request can wait in its queue.
+	newReqWaitCtxFn func(context.Context) (context.Context, context.CancelFunc)
 }
 
 func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Request) {
@@ -240,8 +245,9 @@ func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Reque
 				resultCh <- err
 			}()
 
-			// We create handleCtx with explicit cancelation function.
-			// The reason for it is that Handle() underneath may start additional goroutine
+			// We create handleCtx with an adjusted deadline, for two reasons.
+			// One is to limit the time the request waits before its execution starts.
+			// The other reason for it is that Handle() underneath may start additional goroutine
 			// that is blocked on context cancellation. However, from APF point of view,
 			// we don't want to wait until the whole watch request is processed (which is
 			// when it context is actually cancelled) - we want to unblock the goroutine as
@@ -249,7 +255,7 @@ func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Reque
 			//
 			// Note that we explicitly do NOT call the actuall handler using that context
 			// to avoid cancelling request too early.
-			handleCtx, handleCtxCancel := context.WithCancel(ctx)
+			handleCtx, handleCtxCancel := h.newReqWaitCtxFn(ctx)
 			defer handleCtxCancel()
 
 			// Note that Handle will return irrespective of whether the request
@@ -286,7 +292,11 @@ func (h *priorityAndFairnessHandler) Handle(w http.ResponseWriter, r *http.Reque
 			h.handler.ServeHTTP(w, r)
 		}
 
-		h.fcIfc.Handle(ctx, digest, noteFn, estimateWork, queueNote, execute)
+		func() {
+			handleCtx, cancelFn := h.newReqWaitCtxFn(ctx)
+			defer cancelFn()
+			h.fcIfc.Handle(handleCtx, digest, noteFn, estimateWork, queueNote, execute)
+		}()
 	}
 
 	if !served {
@@ -309,6 +319,7 @@ func WithPriorityAndFairness(
 	longRunningRequestCheck apirequest.LongRunningRequestCheck,
 	fcIfc utilflowcontrol.Interface,
 	workEstimator flowcontrolrequest.WorkEstimatorFunc,
+	defaultRequestWaitLimit time.Duration,
 ) http.Handler {
 	if fcIfc == nil {
 		klog.Warningf("priority and fairness support not found, skipping")
@@ -322,12 +333,18 @@ func WithPriorityAndFairness(
 		waitingMark.mutatingObserver = fcmetrics.GetWaitingMutatingConcurrency()
 	})
 
+	clock := &utilsclock.RealClock{}
+	newReqWaitCtxFn := func(ctx context.Context) (context.Context, context.CancelFunc) {
+		return getRequestWaitContext(ctx, defaultRequestWaitLimit, clock)
+	}
+
 	priorityAndFairnessHandler := &priorityAndFairnessHandler{
 		handler:                 handler,
 		longRunningRequestCheck: longRunningRequestCheck,
 		fcIfc:                   fcIfc,
 		workEstimator:           workEstimator,
 		droppedRequests:         utilflowcontrol.NewDroppedRequestsTracker(),
+		newReqWaitCtxFn:         newReqWaitCtxFn,
 	}
 	return http.HandlerFunc(priorityAndFairnessHandler.Handle)
 }
@@ -356,3 +373,48 @@ func tooManyRequests(req *http.Request, w http.ResponseWriter, retryAfter string
 	w.Header().Set("Retry-After", retryAfter)
 	http.Error(w, "Too many requests, please try again later.", http.StatusTooManyRequests)
 }
+
+// getRequestWaitContext returns a new context with a deadline of how
+// long the request is allowed to wait before it is removed from its
+// queue and rejected.
+// The context.CancelFunc returned must never be nil and the caller is
+// responsible for calling the CancelFunc function for cleanup.
+//   - ctx: the context associated with the request (it may or may
+//     not have a deadline).
+//   - defaultRequestWaitLimit: the default wait duration that is used
+//     if the request context does not have any deadline.
+//     (a) initialization of a watch or
+//     (b) a request whose context has no deadline
+//
+// clock comes in handy for testing the function
+func getRequestWaitContext(ctx context.Context, defaultRequestWaitLimit time.Duration, clock utilsclock.PassiveClock) (context.Context, context.CancelFunc) {
+	if ctx.Err() != nil {
+		return ctx, func() {}
+	}
+
+	reqArrivedAt := clock.Now()
+	if reqReceivedTimestamp, ok := apirequest.ReceivedTimestampFrom(ctx); ok {
+		reqArrivedAt = reqReceivedTimestamp
+	}
+
+	// a) we will allow the request to wait in the queue for one
+	// fourth of the time of its allotted deadline.
+	// b) if the request context does not have any deadline
+	// then we default to 'defaultRequestWaitLimit'
+	// in any case, the wait limit for any request must not
+	// exceed the hard limit of 1m
+	//
+	// request has deadline:
+	//   wait-limit = min(remaining deadline / 4, 1m)
+	// request has no deadline:
+	//   wait-limit = min(defaultRequestWaitLimit, 1m)
+	thisReqWaitLimit := defaultRequestWaitLimit
+	if deadline, ok := ctx.Deadline(); ok {
+		thisReqWaitLimit = deadline.Sub(reqArrivedAt) / 4
+	}
+	if thisReqWaitLimit > time.Minute {
+		thisReqWaitLimit = time.Minute
+	}
+
+	return context.WithDeadline(ctx, reqArrivedAt.Add(thisReqWaitLimit))
+}

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -158,7 +158,7 @@ type GenericAPIServer struct {
 	openAPIConfig *openapicommon.Config
 
 	// Enable swagger and/or OpenAPI V3 if these configs are non-nil.
-	openAPIV3Config *openapicommon.Config
+	openAPIV3Config *openapicommon.OpenAPIV3Config
 
 	// SkipOpenAPIInstallation indicates not to install the OpenAPI handler
 	// during PrepareRun.
@@ -430,11 +430,9 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 	}
 
 	if s.openAPIV3Config != nil && !s.skipOpenAPIInstallation {
-		if utilfeature.DefaultFeatureGate.Enabled(features.OpenAPIV3) {
-			s.OpenAPIV3VersionedService = routes.OpenAPI{
-				Config: s.openAPIV3Config,
-			}.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
-		}
+		s.OpenAPIV3VersionedService = routes.OpenAPI{
+			V3Config: s.openAPIV3Config,
+		}.InstallV3(s.Handler.GoRestfulContainer, s.Handler.NonGoRestfulMux)
 	}
 
 	s.installHealthz()

vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -205,7 +205,6 @@ func StatusIsNot(statuses ...int) StacktracePred {
 func (rl *respLogger) Addf(format string, data ...interface{}) {
 	rl.mutex.Lock()
 	defer rl.mutex.Unlock()
-	rl.addedInfo.WriteString("\n")
 	rl.addedInfo.WriteString(fmt.Sprintf(format, data...))
 }
 

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -42,6 +42,9 @@ func NewAPIEnablementOptions() *APIEnablementOptions {
 
 // AddFlags adds flags for a specific APIServer to the specified FlagSet
 func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
 	fs.Var(&s.RuntimeConfig, "runtime-config", ""+
 		"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
 		"v1=true|false for the core API group\n"+
@@ -87,7 +90,6 @@ func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
 
 // ApplyTo override MergedResourceConfig with defaults and registry
 func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
-
 	if s == nil {
 		return nil
 	}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -105,10 +105,36 @@ const (
 	kmsReloadHealthCheckName = "kms-providers"
 )
 
+var codecs serializer.CodecFactory
+
+// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests
+// as the feature gate is now locked to true starting with v1.29
+// Note: it cannot be set by an end user
+var kdfDisabled atomic.Bool
+
+// this function should only be called in tests to swap enablement of the KMSv2KDF feature
+func SetKDFForTests(b bool) func() {
+	kdfDisabled.Store(!b)
+	return func() {
+		kdfDisabled.Store(false)
+	}
+}
+
+// this function should be used to determine enablement of the KMSv2KDF feature
+// instead of getting it from DefaultFeatureGate as the feature gate is now locked
+// to true starting with v1.29
+func GetKDF() bool {
+	return !kdfDisabled.Load()
+}
+
 func init() {
-	metrics.RegisterMetrics()
-	storagevalue.RegisterMetrics()
+	configScheme := runtime.NewScheme()
+	utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
+	utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
+	codecs = serializer.NewCodecFactory(configScheme)
 	envelopemetrics.RegisterMetrics()
+	storagevalue.RegisterMetrics()
+	metrics.RegisterMetrics()
 }
 
 type kmsPluginHealthzResponse struct {
@@ -131,6 +157,8 @@ type kmsv2PluginProbe struct {
 	service      kmsservice.Service
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
+	apiServerID  string
+	version      string
 }
 
 type kmsHealthChecker []healthz.HealthChecker
@@ -184,13 +212,13 @@ type EncryptionConfiguration struct {
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
 // If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
-func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*EncryptionConfiguration, error) {
+func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
 	config, contentHash, err := loadConfig(filepath, reload)
 	if err != nil {
 		return nil, fmt.Errorf("error while parsing file: %w", err)
 	}
 
-	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config)
+	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
 	if err != nil {
 		return nil, fmt.Errorf("error while building transformers: %w", err)
 	}
@@ -215,9 +243,9 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
 // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
 	var kmsHealthChecks []healthz.HealthChecker
-	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
+	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -236,7 +264,7 @@ type healthChecker interface {
 // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
 	resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -245,7 +273,7 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
 	for _, resourceConfig := range config.Resources {
 		resourceConfig := resourceConfig
 
-		transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig)
+		transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
 		if err != nil {
 			return nil, nil, nil, err
 		}
@@ -362,7 +390,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 	// this gate can only change during tests, but the check is cheap enough to always make
 	// this allows us to easily exercise both modes without restarting the API server
 	// TODO integration test that this dynamically takes effect
-	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	useSeed := GetKDF()
 	stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
 
 	// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
@@ -447,15 +475,23 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 	if response.Healthz != "ok" {
 		errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
 	}
-	if response.Version != envelopekmsv2.KMSAPIVersion {
-		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
+	if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 {
+		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version))
+	} else {
+		// set version for the first status response
+		if len(h.version) == 0 {
+			h.version = response.Version
+		}
+		if h.version != response.Version {
+			errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version))
+		}
 	}
 
 	if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
 		envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
 		errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
 	} else {
-		envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID)
+		envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
 		// unconditionally append as we filter out nil errors below
 		errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
 	}
@@ -468,6 +504,24 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 
 // loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
 func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
+	data, contentHash, err := loadDataAndHash(filepath)
+	if err != nil {
+		return nil, "", fmt.Errorf("error while loading file: %w", err)
+	}
+
+	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
+	}
+	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	if !ok {
+		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+
+	return config, contentHash, validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
+}
+
+func loadDataAndHash(filepath string) ([]byte, string, error) {
 	f, err := os.Open(filepath)
 	if err != nil {
 		return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
@@ -482,27 +536,20 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
 		return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
 	}
 
-	scheme := runtime.NewScheme()
-	codecs := serializer.NewCodecFactory(scheme)
-	utilruntime.Must(apiserverconfig.AddToScheme(scheme))
-	utilruntime.Must(apiserverconfigv1.AddToScheme(scheme))
+	return data, computeEncryptionConfigHash(data), nil
+}
 
-	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
-	}
-	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
-	if !ok {
-		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
-	}
-
-	return config, computeEncryptionConfigHash(data), validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
+// GetEncryptionConfigHash reads the encryption configuration file at filepath and returns the hash of the file.
+// It does not attempt to decode or load the config, and serves as a cheap check to determine if the file has changed.
+func GetEncryptionConfigHash(filepath string) (string, error) {
+	_, contentHash, err := loadDataAndHash(filepath)
+	return contentHash, err
 }
 
 // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
+func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
 	var transformers []storagevalue.PrefixTransformer
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -530,7 +577,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 			transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
 
 		case provider.KMS != nil:
-			transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS)
+			transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
 			if transformerErr == nil {
 				probes = append(probes, probe)
 				kmsUsed.accumulate(used)
@@ -689,7 +736,7 @@ func (s *kmsState) accumulate(other *kmsState) {
 // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
+func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
 	kmsName := config.Name
 	switch config.APIVersion {
 	case kmsAPIVersionV1:
@@ -735,14 +782,14 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 			service:      envelopeService,
 			l:            &sync.Mutex{},
 			lastResponse: &kmsPluginHealthzResponse{},
+			apiServerID:  apiServerID,
 		}
 		// initialize state so that Load always works
 		probe.state.Store(&envelopekmsv2.State{})
 
 		primeAndProbeKMSv2(ctx, probe, kmsName)
-
 		transformer := storagevalue.PrefixTransformer{
-			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState),
+			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
 			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
 		}
 

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -20,9 +20,9 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -35,8 +35,11 @@ import (
 // workqueueKey is the dummy key used to process change in encryption config file.
 const workqueueKey = "key"
 
-// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file.
-type DynamicKMSEncryptionConfigContent struct {
+// EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed.
+var EncryptionConfigFileChangePollDuration = time.Minute
+
+// DynamicEncryptionConfigContent which can dynamically handle changes in encryption config file.
+type DynamicEncryptionConfigContent struct {
 	name string
 
 	// filePath is the path of the file to read.
@@ -50,6 +53,17 @@ type DynamicKMSEncryptionConfigContent struct {
 
 	// dynamicTransformers updates the transformers when encryption config file changes.
 	dynamicTransformers *encryptionconfig.DynamicTransformers
+
+	// identity of the api server
+	apiServerID string
+
+	// can be swapped during testing
+	getEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
+	loadEncryptionConfig    func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
+}
+
+func init() {
+	metrics.RegisterMetrics()
 }
 
 // NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
@@ -57,94 +71,73 @@ func NewDynamicEncryptionConfiguration(
 	name, filePath string,
 	dynamicTransformers *encryptionconfig.DynamicTransformers,
 	configContentHash string,
-) *DynamicKMSEncryptionConfigContent {
-	encryptionConfig := &DynamicKMSEncryptionConfigContent{
+	apiServerID string,
+) *DynamicEncryptionConfigContent {
+	return &DynamicEncryptionConfigContent{
 		name:                           name,
 		filePath:                       filePath,
 		lastLoadedEncryptionConfigHash: configContentHash,
 		dynamicTransformers:            dynamicTransformers,
 		queue:                          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
+		apiServerID:                    apiServerID,
+		getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
+			return encryptionconfig.GetEncryptionConfigHash(filepath)
+		},
+		loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig,
 	}
-	encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run
-
-	return encryptionConfig
 }
 
-// Run starts the controller and blocks until stopCh is closed.
-func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
+// Run starts the controller and blocks until ctx is canceled.
+func (d *DynamicEncryptionConfigContent) Run(ctx context.Context) {
 	defer utilruntime.HandleCrash()
-	defer d.queue.ShutDown()
 
 	klog.InfoS("Starting controller", "name", d.name)
 	defer klog.InfoS("Shutting down controller", "name", d.name)
 
-	// start worker for processing content
-	go wait.UntilWithContext(ctx, d.runWorker, time.Second)
+	var wg sync.WaitGroup
 
-	// start the loop that watches the encryption config file until stopCh is closed.
-	go wait.UntilWithContext(ctx, func(ctx context.Context) {
-		if err := d.watchEncryptionConfigFile(ctx); err != nil {
-			// if there is an error while setting up or handling the watches, this will ensure that we will process the config file.
-			defer d.queue.Add(workqueueKey)
-			klog.ErrorS(err, "Failed to watch encryption config file, will retry later")
-		}
-	}, time.Second)
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		defer d.queue.ShutDown()
+		<-ctx.Done()
+	}()
 
-	<-ctx.Done()
-}
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		d.runWorker(ctx)
+	}()
 
-func (d *DynamicKMSEncryptionConfigContent) watchEncryptionConfigFile(ctx context.Context) error {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return fmt.Errorf("error creating fsnotify watcher: %w", err)
-	}
-	defer watcher.Close()
+	// this function polls changes in the encryption config file by placing a dummy key in the queue.
+	// the 'runWorker' function then picks up this dummy key and processes the changes.
+	// the goroutine terminates when 'ctx' is canceled.
+	_ = wait.PollUntilContextCancel(
+		ctx,
+		EncryptionConfigFileChangePollDuration,
+		true,
+		func(ctx context.Context) (bool, error) {
+			// add dummy item to the queue to trigger file content processing.
+			d.queue.Add(workqueueKey)
 
-	if err = watcher.Add(d.filePath); err != nil {
-		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
-	}
+			// return false to continue polling.
+			return false, nil
+		},
+	)
 
-	for {
-		select {
-		case event := <-watcher.Events:
-			if err := d.handleWatchEvent(event, watcher); err != nil {
-				return err
-			}
-		case err := <-watcher.Errors:
-			return fmt.Errorf("received fsnotify error: %w", err)
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (d *DynamicKMSEncryptionConfigContent) handleWatchEvent(event fsnotify.Event, watcher *fsnotify.Watcher) error {
-	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
-	defer d.queue.Add(workqueueKey)
-
-	// return if file has not been removed or renamed.
-	if event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
-		return nil
-	}
-
-	if err := watcher.Remove(d.filePath); err != nil {
-		klog.V(2).InfoS("Failed to remove file watch, it may have been deleted", "file", d.filePath, "err", err)
-	}
-	if err := watcher.Add(d.filePath); err != nil {
-		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
-	}
-
-	return nil
+	wg.Wait()
 }
 
 // runWorker to process file content
-func (d *DynamicKMSEncryptionConfigContent) runWorker(ctx context.Context) {
+func (d *DynamicEncryptionConfigContent) runWorker(ctx context.Context) {
 	for d.processNextWorkItem(ctx) {
 	}
 }
 
 // processNextWorkItem processes file content when there is a message in the queue.
-func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
+func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
 	// key here is dummy item in the queue to trigger file content processing.
 	key, quit := d.queue.Get()
 	if quit {
@@ -152,6 +145,12 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	}
 	defer d.queue.Done(key)
 
+	d.processWorkItem(serverCtx, key)
+
+	return true
+}
+
+func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
 	var (
 		updatedEffectiveConfig  bool
 		err                     error
@@ -172,32 +171,32 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 		}
 
 		if updatedEffectiveConfig && err == nil {
-			metrics.RecordEncryptionConfigAutomaticReloadSuccess()
+			metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
 		}
 
 		if err != nil {
-			metrics.RecordEncryptionConfigAutomaticReloadFailure()
+			metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
 			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
 			// add dummy item back to the queue to trigger file content processing.
-			d.queue.AddRateLimited(key)
+			d.queue.AddRateLimited(workqueueKey)
 		}
 	}()
 
 	encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
 	if err != nil {
-		return true
+		return
 	}
 	if !configChanged {
-		return true
+		return
 	}
 
 	if len(encryptionConfiguration.HealthChecks) != 1 {
 		err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
-		return true
+		return
 	}
 	// get healthz checks for all new KMS plugins.
 	if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
-		return true
+		return
 	}
 
 	// update transformers.
@@ -214,30 +213,44 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
 
 	updatedEffectiveConfig = true
-	return true
 }
 
 // loadEncryptionConfig processes the next set of content from the file.
-func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
-	encryptionConfiguration *encryptionconfig.EncryptionConfiguration,
+func (d *DynamicEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
+	_ *encryptionconfig.EncryptionConfiguration,
 	configChanged bool,
-	err error,
+	_ error,
 ) {
-	// this code path will only execute if reload=true. So passing true explicitly.
-	encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true)
+	contentHash, err := d.getEncryptionConfigHash(ctx, d.filePath)
 	if err != nil {
 		return nil, false, err
 	}
 
 	// check if encryptionConfig is different from the current. Do nothing if they are the same.
-	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
-		klog.V(4).InfoS("Encryption config has not changed", "name", d.name)
+	if contentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (before load)", "name", d.name)
 		return nil, false, nil
 	}
+
+	// this code path will only execute if reload=true. So passing true explicitly.
+	encryptionConfiguration, err := d.loadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// check if encryptionConfig is different from the current (again to avoid TOCTOU). Do nothing if they are the same.
+	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (after load)", "name", d.name)
+		return nil, false, nil
+	}
+
 	return encryptionConfiguration, true, nil
 }
 
-func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
+// minKMSPluginCloseGracePeriod can be lowered in unit tests to make the health check poll faster
+var minKMSPluginCloseGracePeriod = 10 * time.Second
+
+func (d *DynamicEncryptionConfigContent) validateNewTransformersHealth(
 	ctx context.Context,
 	kmsPluginHealthzCheck healthz.HealthChecker,
 	kmsPluginCloseGracePeriod time.Duration,
@@ -245,8 +258,8 @@ func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
 	// test if new transformers are healthy
 	var healthCheckError error
 
-	if kmsPluginCloseGracePeriod < 10*time.Second {
-		kmsPluginCloseGracePeriod = 10 * time.Second
+	if kmsPluginCloseGracePeriod < minKMSPluginCloseGracePeriod {
+		kmsPluginCloseGracePeriod = minKMSPluginCloseGracePeriod
 	}
 
 	// really make sure that the immediate check does not hang

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -17,6 +17,9 @@ limitations under the License.
 package metrics
 
 import (
+	"crypto/sha256"
+	"fmt"
+	"hash"
 	"sync"
 
 	"k8s.io/component-base/metrics"
@@ -29,24 +32,26 @@ const (
 )
 
 var (
-	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
+	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_failures_total",
-			Help:           "Total number of failed automatic reloads of encryption configuration.",
+			Help:           "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
+		[]string{"apiserver_id_hash"},
 	)
 
-	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
+	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_success_total",
-			Help:           "Total number of successful automatic reloads of encryption configuration.",
+			Help:           "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
+		[]string{"apiserver_id_hash"},
 	)
 
 	encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
@@ -54,33 +59,53 @@ var (
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_last_timestamp_seconds",
-			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration.",
+			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"status"},
+		[]string{"status", "apiserver_id_hash"},
 	)
 )
 
 var registerMetrics sync.Once
+var hashPool *sync.Pool
 
 func RegisterMetrics() {
 	registerMetrics.Do(func() {
+		hashPool = &sync.Pool{
+			New: func() interface{} {
+				return sha256.New()
+			},
+		}
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
 	})
 }
 
-func RecordEncryptionConfigAutomaticReloadFailure() {
-	encryptionConfigAutomaticReloadFailureTotal.Inc()
-	recordEncryptionConfigAutomaticReloadTimestamp("failure")
+func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
 }
 
-func RecordEncryptionConfigAutomaticReloadSuccess() {
-	encryptionConfigAutomaticReloadSuccessTotal.Inc()
-	recordEncryptionConfigAutomaticReloadTimestamp("success")
+func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
 }
 
-func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
-	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
+func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
+	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
+}
+
+func getHash(data string) string {
+	if len(data) == 0 {
+		return ""
+	}
+	h := hashPool.Get().(hash.Hash)
+	h.Reset()
+	h.Write([]byte(data))
+	dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
+	hashPool.Put(h)
+	return dataHash
 }

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -44,8 +45,6 @@ import (
 )
 
 type EtcdOptions struct {
-	// The value of Paging on StorageConfig will be overridden by the
-	// calculated feature gate value.
 	StorageConfig                           storagebackend.Config
 	EncryptionProviderConfigFilepath        string
 	EncryptionProviderConfigAutomaticReload bool
@@ -87,6 +86,12 @@ func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
 	return options
 }
 
+var storageMediaTypes = sets.New(
+	runtime.ContentTypeJSON,
+	runtime.ContentTypeYAML,
+	runtime.ContentTypeProtobuf,
+)
+
 func (s *EtcdOptions) Validate() []error {
 	if s == nil {
 		return nil
@@ -120,6 +125,10 @@ func (s *EtcdOptions) Validate() []error {
 		allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
 	}
 
+	if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) {
+		allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", ")))
+	}
+
 	return allErrors
 }
 
@@ -294,7 +303,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 		}
 	}()
 
-	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload)
+	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
 	if err != nil {
 		return err
 	}
@@ -318,6 +327,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 					s.EncryptionProviderConfigFilepath,
 					dynamicTransformers,
 					encryptionConfiguration.EncryptionFileContentHash,
+					c.APIServerID,
 				)
 
 				go dynamicEncryptionConfigController.Run(ctxServer)
@@ -331,18 +341,23 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 
 		c.ResourceTransformers = dynamicTransformers
 		if !s.SkipHealthEndpoints {
-			c.AddHealthChecks(dynamicTransformers)
+			addHealthChecksWithoutLivez(c, dynamicTransformers)
 		}
 	} else {
 		c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
 		if !s.SkipHealthEndpoints {
-			c.AddHealthChecks(encryptionConfiguration.HealthChecks...)
+			addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...)
 		}
 	}
 
 	return nil
 }
 
+func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) {
+	c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
+	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
+}
+
 func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
 	if err != nil {

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -17,16 +17,22 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
+
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apiserver/pkg/server"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
 )
 
 type FeatureOptions struct {
 	EnableProfiling           bool
 	DebugSocketPath           string
 	EnableContentionProfiling bool
+	EnablePriorityAndFairness bool
 }
 
 func NewFeatureOptions() *FeatureOptions {
@@ -36,6 +42,7 @@ func NewFeatureOptions() *FeatureOptions {
 		EnableProfiling:           defaults.EnableProfiling,
 		DebugSocketPath:           defaults.DebugSocketPath,
 		EnableContentionProfiling: defaults.EnableContentionProfiling,
+		EnablePriorityAndFairness: true,
 	}
 }
 
@@ -50,9 +57,11 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
 		"Enable block profiling, if profiling is enabled")
 	fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath,
 		"Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path")
+	fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+
+		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
 }
 
-func (o *FeatureOptions) ApplyTo(c *server.Config) error {
+func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error {
 	if o == nil {
 		return nil
 	}
@@ -61,6 +70,18 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error {
 	c.DebugSocketPath = o.DebugSocketPath
 	c.EnableContentionProfiling = o.EnableContentionProfiling
 
+	if o.EnablePriorityAndFairness {
+		if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
+			return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
+
+		}
+		c.FlowControl = utilflowcontrol.New(
+			informers,
+			clientset.FlowcontrolV1(),
+			c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight,
+		)
+	}
+
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -17,20 +17,15 @@ limitations under the License.
 package options
 
 import (
-	"fmt"
-
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/util/feature"
-	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/featuregate"
-	"k8s.io/klog/v2"
 )
 
 // RecommendedOptions contains the recommended options for running an API server.
@@ -122,17 +117,17 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Audit.ApplyTo(&config.Config); err != nil {
 		return err
 	}
-	if err := o.Features.ApplyTo(&config.Config); err != nil {
-		return err
-	}
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
-	initializers, err := o.ExtraAdmissionInitializers(config)
+	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
 	if err != nil {
 		return err
 	}
-	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
+	if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
+		return err
+	}
+	initializers, err := o.ExtraAdmissionInitializers(config)
 	if err != nil {
 		return err
 	}
@@ -144,22 +139,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 		initializers...); err != nil {
 		return err
 	}
-	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
-		if config.ClientConfig != nil {
-			if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
-				return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight)
-
-			}
-			config.FlowControl = utilflowcontrol.New(
-				config.SharedInformerFactory,
-				kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(),
-				config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
-				config.RequestTimeout/4,
-			)
-		} else {
-			klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled")
-		}
-	}
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -62,8 +62,7 @@ type ServerRunOptions struct {
 	// decoded in a write request. 0 means no limit.
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
-	MaxRequestBodyBytes       int64
-	EnablePriorityAndFairness bool
+	MaxRequestBodyBytes int64
 
 	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
 	// Server during the graceful termination of the apiserver. If true, we wait
@@ -104,7 +103,6 @@ func NewServerRunOptions() *ServerRunOptions {
 		ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
 		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
-		EnablePriorityAndFairness:           true,
 		ShutdownSendRetryAfter:              false,
 	}
 }
@@ -325,9 +323,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
-	fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
-		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
-
 	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
 		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
 		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -260,7 +260,39 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	c := *config
 
 	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
-	// load main cert
+	// load main cert *original description until 2023-08-18*
+
+	/*
+		kubernetes mutual (2-way) x509 between client and apiserver:
+
+			>1. apiserver sending its apiserver certificate along with its publickey to client
+			2. client verifies the apiserver certificate sent against its cluster certificate authority data
+			3. client sending its client certificate along with its public key to the apiserver
+			4. apiserver verifies the client certificate sent against its cluster certificate authority data
+
+			description:
+				here, with this block,
+				apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo
+				for client to later in the step 2 verify the apiserver certificate during the handshake
+				when making a request
+
+			normal args related to this stage:
+				--tls-cert-file string  File containing the default x509 Certificate for HTTPS.
+					(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and
+					--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate
+					and key are generated for the public address and saved to the directory specified by
+					--cert-dir
+				--tls-private-key-file string  File containing the default x509 private key matching --tls-cert-file.
+
+				(retrievable from "kube-apiserver --help" command)
+				(suggested by @deads2k)
+
+			see also:
+				- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
+	*/
+
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
 		var err error
 		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)

vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -17,6 +17,7 @@ limitations under the License.
 package routes
 
 import (
+	handlersmetrics "k8s.io/apiserver/pkg/endpoints/handlers/metrics"
 	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/mux"
 	cachermetrics "k8s.io/apiserver/pkg/storage/cacher/metrics"
@@ -52,4 +53,5 @@ func register() {
 	etcd3metrics.Register()
 	flowcontrolmetrics.Register()
 	peerproxymetrics.Register()
+	handlersmetrics.Register()
 }

vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -32,7 +32,8 @@ import (
 
 // OpenAPI installs spec endpoints for each web service.
 type OpenAPI struct {
-	Config *common.Config
+	Config   *common.Config
+	V3Config *common.OpenAPIV3Config
 }
 
 // Install adds the SwaggerUI webservice to the given mux.
@@ -65,7 +66,7 @@ func (oa OpenAPI) InstallV3(c *restful.Container, mux *mux.PathRecorderMux) *han
 	}
 
 	for gv, ws := range grouped {
-		spec, err := builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(ws), oa.Config)
+		spec, err := builder3.BuildOpenAPISpecFromRoutes(restfuladapter.AdaptWebServices(ws), oa.V3Config)
 		if err != nil {
 			klog.Errorf("Failed to build OpenAPI v3 for group %s, %q", gv, err)
 

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -19,15 +19,13 @@ package storage
 import (
 	"crypto/tls"
 	"crypto/x509"
-	"io/ioutil"
+	"os"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
-	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 )
 
@@ -114,8 +112,6 @@ type groupResourceOverrides struct {
 	// decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of
 	// returned decoders will be priority for attempt to decode.
 	decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder
-	// disablePaging will prevent paging on the provided resource.
-	disablePaging bool
 }
 
 // Apply overrides the provided config and options if the override has a value in that position
@@ -139,9 +135,6 @@ func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *St
 	if o.decoderDecoratorFn != nil {
 		options.DecoderDecoratorFn = o.decoderDecoratorFn
 	}
-	if o.disablePaging {
-		config.Paging = false
-	}
 }
 
 var _ StorageFactory = &DefaultStorageFactory{}
@@ -156,7 +149,6 @@ func NewDefaultStorageFactory(
 	resourceConfig APIResourceConfigSource,
 	specialDefaultResourcePrefixes map[schema.GroupResource]string,
 ) *DefaultStorageFactory {
-	config.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	if len(defaultMediaType) == 0 {
 		defaultMediaType = runtime.ContentTypeJSON
 	}
@@ -185,14 +177,6 @@ func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource
 	s.Overrides[groupResource] = overrides
 }
 
-// SetDisableAPIListChunking allows a specific resource to disable paging at the storage layer, to prevent
-// exposure of key names in continuations. This may be overridden by feature gates.
-func (s *DefaultStorageFactory) SetDisableAPIListChunking(groupResource schema.GroupResource) {
-	overrides := s.Overrides[groupResource]
-	overrides.disablePaging = true
-	s.Overrides[groupResource] = overrides
-}
-
 // SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir.  You'll end up in `etcdPrefix/resourceEtcdPrefix`.
 func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) {
 	overrides := s.Overrides[groupResource]
@@ -337,7 +321,7 @@ func backends(storageConfig storagebackend.Config, grOverrides map[schema.GroupR
 		}
 	}
 	if len(storageConfig.Transport.TrustedCAFile) > 0 {
-		if caCert, err := ioutil.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
+		if caCert, err := os.ReadFile(storageConfig.Transport.TrustedCAFile); err != nil {
 			klog.Errorf("failed to read ca file while getting backends: %s", err)
 		} else {
 			caPool := x509.NewCertPool()

vendor/k8s.io/apiserver/pkg/storage/cacher/cache_watcher.go

@@ -22,7 +22,6 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -177,7 +176,6 @@ func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool {
 		// This means that we couldn't send event to that watcher.
 		// Since we don't want to block on it infinitely,
 		// we simply terminate it.
-		klog.V(1).Infof("Forcing %v watcher close due to unresponsiveness: %v. len(c.input) = %v, len(c.result) = %v", c.groupResource.String(), c.identifier, len(c.input), len(c.result))
 		metrics.TerminatedWatchersCounter.WithLabelValues(c.groupResource.String()).Inc()
 		// This means that we couldn't send event to that watcher.
 		// Since we don't want to block on it infinitely, we simply terminate it.
@@ -365,17 +363,10 @@ func (c *cacheWatcher) convertToWatchEvent(event *watchCacheEvent) *watch.Event
 	if event.Type == watch.Bookmark {
 		e := &watch.Event{Type: watch.Bookmark, Object: event.Object.DeepCopyObject()}
 		if !c.wasBookmarkAfterRvSent() {
-			objMeta, err := meta.Accessor(e.Object)
-			if err != nil {
+			if err := storage.AnnotateInitialEventsEndBookmark(e.Object); err != nil {
 				utilruntime.HandleError(fmt.Errorf("error while accessing object's metadata gr: %v, identifier: %v, obj: %#v, err: %v", c.groupResource, c.identifier, e.Object, err))
 				return nil
 			}
-			objAnnotations := objMeta.GetAnnotations()
-			if objAnnotations == nil {
-				objAnnotations = map[string]string{}
-			}
-			objAnnotations["k8s.io/initial-events-end"] = "true"
-			objMeta.SetAnnotations(objAnnotations)
 		}
 		return e
 	}

vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net/http"
 	"reflect"
-	"strconv"
 	"sync"
 	"time"
 
@@ -113,11 +112,8 @@ func (wm watchersMap) addWatcher(w *cacheWatcher, number int) {
 	wm[number] = w
 }
 
-func (wm watchersMap) deleteWatcher(number int, done func(*cacheWatcher)) {
-	if watcher, ok := wm[number]; ok {
-		delete(wm, number)
-		done(watcher)
-	}
+func (wm watchersMap) deleteWatcher(number int) {
+	delete(wm, number)
 }
 
 func (wm watchersMap) terminateAll(done func(*cacheWatcher)) {
@@ -148,14 +144,14 @@ func (i *indexedWatchers) addWatcher(w *cacheWatcher, number int, scope namespac
 	}
 }
 
-func (i *indexedWatchers) deleteWatcher(number int, scope namespacedName, value string, supported bool, done func(*cacheWatcher)) {
+func (i *indexedWatchers) deleteWatcher(number int, scope namespacedName, value string, supported bool) {
 	if supported {
-		i.valueWatchers[value].deleteWatcher(number, done)
+		i.valueWatchers[value].deleteWatcher(number)
 		if len(i.valueWatchers[value]) == 0 {
 			delete(i.valueWatchers, value)
 		}
 	} else {
-		i.allWatchers[scope].deleteWatcher(number, done)
+		i.allWatchers[scope].deleteWatcher(number)
 		if len(i.allWatchers[scope]) == 0 {
 			delete(i.allWatchers, scope)
 		}
@@ -725,15 +721,14 @@ func shouldDelegateList(opts storage.ListOptions) bool {
 	resourceVersion := opts.ResourceVersion
 	pred := opts.Predicate
 	match := opts.ResourceVersionMatch
-	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	consistentListFromCacheEnabled := utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache)
 
 	// Serve consistent reads from storage if ConsistentListFromCache is disabled
 	consistentReadFromStorage := resourceVersion == "" && !consistentListFromCacheEnabled
 	// Watch cache doesn't support continuations, so serve them from etcd.
-	hasContinuation := pagingEnabled && len(pred.Continue) > 0
+	hasContinuation := len(pred.Continue) > 0
 	// Serve paginated requests about revision "0" from watch cache to avoid overwhelming etcd.
-	hasLimit := pagingEnabled && pred.Limit > 0 && resourceVersion != "0"
+	hasLimit := pred.Limit > 0 && resourceVersion != "0"
 	// Watch cache only supports ResourceVersionMatchNotOlderThan (default).
 	unsupportedMatch := match != "" && match != metav1.ResourceVersionMatchNotOlderThan
 
@@ -773,7 +768,7 @@ func (c *Cacher) GetList(ctx context.Context, key string, opts storage.ListOptio
 		return c.storage.GetList(ctx, key, opts, listObj)
 	}
 	if listRV == 0 && utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCache) {
-		listRV, err = c.getCurrentResourceVersionFromStorage(ctx)
+		listRV, err = storage.GetCurrentResourceVersionFromStorage(ctx, c.storage, c.newListFunc, c.resourcePrefix, c.objectType.String())
 		if err != nil {
 			return err
 		}
@@ -1225,7 +1220,8 @@ func forgetWatcher(c *Cacher, w *cacheWatcher, index int, scope namespacedName,
 		// It's possible that the watcher is already not in the structure (e.g. in case of
 		// simultaneous Stop() and terminateAllWatchers(), but it is safe to call stopLocked()
 		// on a watcher multiple times.
-		c.watchers.deleteWatcher(index, scope, triggerValue, triggerSupported, c.stopWatcherLocked)
+		c.watchers.deleteWatcher(index, scope, triggerValue, triggerSupported)
+		c.stopWatcherLocked(w)
 	}
 }
 
@@ -1249,48 +1245,12 @@ func (c *Cacher) LastSyncResourceVersion() (uint64, error) {
 	return c.versioner.ParseResourceVersion(resourceVersion)
 }
 
-// getCurrentResourceVersionFromStorage gets the current resource version from the underlying storage engine.
-// this method issues an empty list request and reads only the ResourceVersion from the object metadata
-func (c *Cacher) getCurrentResourceVersionFromStorage(ctx context.Context) (uint64, error) {
-	if c.newListFunc == nil {
-		return 0, fmt.Errorf("newListFunction wasn't provided for %v", c.objectType)
-	}
-	emptyList := c.newListFunc()
-	pred := storage.SelectionPredicate{
-		Label: labels.Everything(),
-		Field: fields.Everything(),
-		Limit: 1, // just in case we actually hit something
-	}
-
-	err := c.storage.GetList(ctx, c.resourcePrefix, storage.ListOptions{Predicate: pred}, emptyList)
-	if err != nil {
-		return 0, err
-	}
-	emptyListAccessor, err := meta.ListAccessor(emptyList)
-	if err != nil {
-		return 0, err
-	}
-	if emptyListAccessor == nil {
-		return 0, fmt.Errorf("unable to extract a list accessor from %T", emptyList)
-	}
-
-	currentResourceVersion, err := strconv.Atoi(emptyListAccessor.GetResourceVersion())
-	if err != nil {
-		return 0, err
-	}
-
-	if currentResourceVersion == 0 {
-		return 0, fmt.Errorf("the current resource version must be greater than 0")
-	}
-	return uint64(currentResourceVersion), nil
-}
-
 // getBookmarkAfterResourceVersionLockedFunc returns a function that
 // spits a ResourceVersion after which the bookmark event will be delivered.
 //
 // The returned function must be called under the watchCache lock.
 func (c *Cacher) getBookmarkAfterResourceVersionLockedFunc(ctx context.Context, parsedResourceVersion uint64, opts storage.ListOptions) (func() uint64, error) {
-	if opts.SendInitialEvents == nil || *opts.SendInitialEvents == false || !opts.Predicate.AllowWatchBookmarks {
+	if opts.SendInitialEvents == nil || !*opts.SendInitialEvents || !opts.Predicate.AllowWatchBookmarks {
 		return func() uint64 { return 0 }, nil
 	}
 	return c.getCommonResourceVersionLockedFunc(ctx, parsedResourceVersion, opts)
@@ -1305,7 +1265,7 @@ func (c *Cacher) getBookmarkAfterResourceVersionLockedFunc(ctx context.Context,
 //
 // The returned function must be called under the watchCache lock.
 func (c *Cacher) getStartResourceVersionForWatchLockedFunc(ctx context.Context, parsedWatchResourceVersion uint64, opts storage.ListOptions) (func() uint64, error) {
-	if opts.SendInitialEvents == nil || *opts.SendInitialEvents == true {
+	if opts.SendInitialEvents == nil || *opts.SendInitialEvents {
 		return func() uint64 { return parsedWatchResourceVersion }, nil
 	}
 	return c.getCommonResourceVersionLockedFunc(ctx, parsedWatchResourceVersion, opts)
@@ -1318,7 +1278,7 @@ func (c *Cacher) getStartResourceVersionForWatchLockedFunc(ctx context.Context,
 func (c *Cacher) getCommonResourceVersionLockedFunc(ctx context.Context, parsedWatchResourceVersion uint64, opts storage.ListOptions) (func() uint64, error) {
 	switch {
 	case len(opts.ResourceVersion) == 0:
-		rv, err := c.getCurrentResourceVersionFromStorage(ctx)
+		rv, err := storage.GetCurrentResourceVersionFromStorage(ctx, c.storage, c.newListFunc, c.resourcePrefix, c.objectType.String())
 		if err != nil {
 			return nil, err
 		}
@@ -1336,7 +1296,7 @@ func (c *Cacher) getCommonResourceVersionLockedFunc(ctx context.Context, parsedW
 // Additionally, it instructs the caller whether it should ask for
 // all events from the cache (full state) or not.
 func (c *Cacher) waitUntilWatchCacheFreshAndForceAllEvents(ctx context.Context, requestedWatchRV uint64, opts storage.ListOptions) (bool, error) {
-	if opts.SendInitialEvents != nil && *opts.SendInitialEvents == true {
+	if opts.SendInitialEvents != nil && *opts.SendInitialEvents {
 		err := c.watchCache.waitUntilFreshAndBlock(ctx, requestedWatchRV)
 		defer c.watchCache.RUnlock()
 		return err == nil, err

vendor/k8s.io/apiserver/pkg/storage/errors.go

@@ -17,13 +17,16 @@ limitations under the License.
 package storage
 
 import (
+	"errors"
 	"fmt"
 
-	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
+var ErrResourceVersionSetOnCreate = errors.New("resourceVersion should not be set on objects to be created")
+
 const (
 	ErrCodeKeyNotFound int = iota + 1
 	ErrCodeKeyExists
@@ -176,7 +179,7 @@ var tooLargeResourceVersionCauseMsg = "Too large resource version"
 // NewTooLargeResourceVersionError returns a timeout error with the given retrySeconds for a request for
 // a minimum resource version that is larger than the largest currently available resource version for a requested resource.
 func NewTooLargeResourceVersionError(minimumResourceVersion, currentRevision uint64, retrySeconds int) error {
-	err := errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %d, current: %d", minimumResourceVersion, currentRevision), retrySeconds)
+	err := apierrors.NewTimeoutError(fmt.Sprintf("Too large resource version: %d, current: %d", minimumResourceVersion, currentRevision), retrySeconds)
 	err.ErrStatus.Details.Causes = []metav1.StatusCause{
 		{
 			Type:    metav1.CauseTypeResourceVersionTooLarge,
@@ -188,8 +191,8 @@ func NewTooLargeResourceVersionError(minimumResourceVersion, currentRevision uin
 
 // IsTooLargeResourceVersion returns true if the error is a TooLargeResourceVersion error.
 func IsTooLargeResourceVersion(err error) bool {
-	if !errors.IsTimeout(err) {
+	if !apierrors.IsTimeout(err) {
 		return false
 	}
-	return errors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge)
+	return apierrors.HasStatusCause(err, metav1.CauseTypeResourceVersionTooLarge)
 }

vendor/k8s.io/apiserver/pkg/storage/etcd3/event.go

@@ -30,6 +30,17 @@ type event struct {
 	isDeleted        bool
 	isCreated        bool
 	isProgressNotify bool
+	// isInitialEventsEndBookmark helps us keep track
+	// of whether we have sent an annotated bookmark event.
+	//
+	// when this variable is set to true,
+	// a special annotation will be added
+	// to the bookmark event.
+	//
+	// note that we decided to extend the event
+	// struct field to eliminate contention
+	// between startWatching and processEvent
+	isInitialEventsEndBookmark bool
 }
 
 // parseKV converts a KeyValue retrieved from an initial sync() listing to a synthetic isCreated event.

vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go

@@ -69,7 +69,7 @@ var (
 	objectCounts = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
 			Name:           "apiserver_storage_objects",
-			Help:           "Number of stored objects at the time of last check split by kind.",
+			Help:           "Number of stored objects at the time of last check split by kind. In case of a fetching error, the value will be -1.",
 			StabilityLevel: compbasemetrics.STABLE,
 		},
 		[]string{"resource"},
@@ -228,7 +228,7 @@ func UpdateEtcdDbSize(ep string, size int64) {
 
 // SetStorageMonitorGetter sets monitor getter to allow monitoring etcd stats.
 func SetStorageMonitorGetter(getter func() ([]Monitor, error)) {
-	storageMonitor.monitorGetter = getter
+	storageMonitor.setGetter(getter)
 }
 
 // UpdateLeaseObjectCount sets the etcd_lease_object_counts metric.
@@ -258,9 +258,22 @@ type StorageMetrics struct {
 type monitorCollector struct {
 	compbasemetrics.BaseStableCollector
 
+	mutex         sync.Mutex
 	monitorGetter func() ([]Monitor, error)
 }
 
+func (m *monitorCollector) setGetter(monitorGetter func() ([]Monitor, error)) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	m.monitorGetter = monitorGetter
+}
+
+func (m *monitorCollector) getGetter() func() ([]Monitor, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+	return m.monitorGetter
+}
+
 // DescribeWithStability implements compbasemetrics.StableColletor
 func (c *monitorCollector) DescribeWithStability(ch chan<- *compbasemetrics.Desc) {
 	ch <- storageSizeDescription
@@ -268,7 +281,7 @@ func (c *monitorCollector) DescribeWithStability(ch chan<- *compbasemetrics.Desc
 
 // CollectWithStability implements compbasemetrics.StableColletor
 func (c *monitorCollector) CollectWithStability(ch chan<- compbasemetrics.Metric) {
-	monitors, err := c.monitorGetter()
+	monitors, err := c.getGetter()()
 	if err != nil {
 		return
 	}

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -32,19 +32,15 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/util/validation/field"
-
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/conversion"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/audit"
-	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/value"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/component-base/tracing"
 	"k8s.io/klog/v2"
 )
@@ -81,7 +77,6 @@ type store struct {
 	groupResource       schema.GroupResource
 	groupResourceString string
 	watcher             *watcher
-	pagingEnabled       bool
 	leaseManager        *leaseManager
 }
 
@@ -100,11 +95,11 @@ type objState struct {
 }
 
 // New returns an etcd3 implementation of storage.Interface.
-func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, groupResource schema.GroupResource, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) storage.Interface {
-	return newStore(c, codec, newFunc, prefix, groupResource, transformer, pagingEnabled, leaseManagerConfig)
+func New(c *clientv3.Client, codec runtime.Codec, newFunc, newListFunc func() runtime.Object, prefix, resourcePrefix string, groupResource schema.GroupResource, transformer value.Transformer, leaseManagerConfig LeaseManagerConfig) storage.Interface {
+	return newStore(c, codec, newFunc, newListFunc, prefix, resourcePrefix, groupResource, transformer, leaseManagerConfig)
 }
 
-func newStore(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, groupResource schema.GroupResource, transformer value.Transformer, pagingEnabled bool, leaseManagerConfig LeaseManagerConfig) *store {
+func newStore(c *clientv3.Client, codec runtime.Codec, newFunc, newListFunc func() runtime.Object, prefix, resourcePrefix string, groupResource schema.GroupResource, transformer value.Transformer, leaseManagerConfig LeaseManagerConfig) *store {
 	versioner := storage.APIObjectVersioner{}
 	// for compatibility with etcd2 impl.
 	// no-op for default prefix of '/registry'.
@@ -114,19 +109,36 @@ func newStore(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Ob
 		// Ensure the pathPrefix ends in "/" here to simplify key concatenation later.
 		pathPrefix += "/"
 	}
-	result := &store{
+
+	w := &watcher{
+		client:        c,
+		codec:         codec,
+		newFunc:       newFunc,
+		groupResource: groupResource,
+		versioner:     versioner,
+		transformer:   transformer,
+	}
+	if newFunc == nil {
+		w.objectType = "<unknown>"
+	} else {
+		w.objectType = reflect.TypeOf(newFunc()).String()
+	}
+	s := &store{
 		client:              c,
 		codec:               codec,
 		versioner:           versioner,
 		transformer:         transformer,
-		pagingEnabled:       pagingEnabled,
 		pathPrefix:          pathPrefix,
 		groupResource:       groupResource,
 		groupResourceString: groupResource.String(),
-		watcher:             newWatcher(c, codec, groupResource, newFunc, versioner),
+		watcher:             w,
 		leaseManager:        newDefaultLeaseManager(c, leaseManagerConfig),
 	}
-	return result
+
+	w.getCurrentStorageRV = func(ctx context.Context) (uint64, error) {
+		return storage.GetCurrentResourceVersionFromStorage(ctx, s, newListFunc, resourcePrefix, w.objectType)
+	}
+	return s
 }
 
 // Versioner implements storage.Interface.Versioner.
@@ -185,7 +197,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
 	)
 	defer span.End(500 * time.Millisecond)
 	if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 {
-		return errors.New("resourceVersion should not be set on objects to be created")
+		return storage.ErrResourceVersionSetOnCreate
 	}
 	if err := s.versioner.PrepareObjectForStorage(obj); err != nil {
 		return fmt.Errorf("PrepareObjectForStorage failed: %v", err)
@@ -258,15 +270,7 @@ func (s *store) Delete(
 func (s *store) conditionalDelete(
 	ctx context.Context, key string, out runtime.Object, v reflect.Value, preconditions *storage.Preconditions,
 	validateDeletion storage.ValidateObjectFunc, cachedExistingObject runtime.Object) error {
-	getCurrentState := func() (*objState, error) {
-		startTime := time.Now()
-		getResp, err := s.client.KV.Get(ctx, key)
-		metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
-		if err != nil {
-			return nil, err
-		}
-		return s.getState(ctx, getResp, key, v, false)
-	}
+	getCurrentState := s.getCurrentState(ctx, key, v, false)
 
 	var origState *objState
 	var err error
@@ -394,15 +398,7 @@ func (s *store) GuaranteedUpdate(
 		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 
-	getCurrentState := func() (*objState, error) {
-		startTime := time.Now()
-		getResp, err := s.client.KV.Get(ctx, preparedKey)
-		metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
-		if err != nil {
-			return nil, err
-		}
-		return s.getState(ctx, getResp, preparedKey, v, ignoreNotFound)
-	}
+	getCurrentState := s.getCurrentState(ctx, preparedKey, v, ignoreNotFound)
 
 	var origState *objState
 	var origStateIsCurrent bool
@@ -594,17 +590,13 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 	if err != nil {
 		return err
 	}
-	recursive := opts.Recursive
-	resourceVersion := opts.ResourceVersion
-	match := opts.ResourceVersionMatch
-	pred := opts.Predicate
-	ctx, span := tracing.Start(ctx, fmt.Sprintf("List(recursive=%v) etcd3", recursive),
+	ctx, span := tracing.Start(ctx, fmt.Sprintf("List(recursive=%v) etcd3", opts.Recursive),
 		attribute.String("audit-id", audit.GetAuditIDTruncated(ctx)),
 		attribute.String("key", key),
-		attribute.String("resourceVersion", resourceVersion),
-		attribute.String("resourceVersionMatch", string(match)),
-		attribute.Int("limit", int(pred.Limit)),
-		attribute.String("continue", pred.Continue))
+		attribute.String("resourceVersion", opts.ResourceVersion),
+		attribute.String("resourceVersionMatch", string(opts.ResourceVersionMatch)),
+		attribute.Int("limit", int(opts.Predicate.Limit)),
+		attribute.String("continue", opts.Predicate.Continue))
 	defer span.End(500 * time.Millisecond)
 	listPtr, err := meta.GetItemsPtr(listObj)
 	if err != nil {
@@ -619,97 +611,68 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 	// get children "directories". e.g. if we have key "/a", "/a/b", "/ab", getting keys
 	// with prefix "/a" will return all three, while with prefix "/a/" will return only
 	// "/a/b" which is the correct answer.
-	if recursive && !strings.HasSuffix(preparedKey, "/") {
+	if opts.Recursive && !strings.HasSuffix(preparedKey, "/") {
 		preparedKey += "/"
 	}
 	keyPrefix := preparedKey
 
 	// set the appropriate clientv3 options to filter the returned data set
 	var limitOption *clientv3.OpOption
-	limit := pred.Limit
+	limit := opts.Predicate.Limit
 	var paging bool
 	options := make([]clientv3.OpOption, 0, 4)
-	if s.pagingEnabled && pred.Limit > 0 {
+	if opts.Predicate.Limit > 0 {
 		paging = true
 		options = append(options, clientv3.WithLimit(limit))
 		limitOption = &options[len(options)-1]
 	}
 
-	newItemFunc := getNewItemFunc(listObj, v)
-
-	var fromRV *uint64
-	if len(resourceVersion) > 0 {
-		parsedRV, err := s.versioner.ParseResourceVersion(resourceVersion)
-		if err != nil {
-			return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
-		}
-		fromRV = &parsedRV
+	if opts.Recursive {
+		rangeEnd := clientv3.GetPrefixRangeEnd(keyPrefix)
+		options = append(options, clientv3.WithRange(rangeEnd))
 	}
 
-	var returnedRV, continueRV, withRev int64
+	newItemFunc := getNewItemFunc(listObj, v)
+
+	var continueRV, withRev int64
 	var continueKey string
 	switch {
-	case recursive && s.pagingEnabled && len(pred.Continue) > 0:
-		continueKey, continueRV, err = storage.DecodeContinue(pred.Continue, keyPrefix)
+	case opts.Recursive && len(opts.Predicate.Continue) > 0:
+		continueKey, continueRV, err = storage.DecodeContinue(opts.Predicate.Continue, keyPrefix)
 		if err != nil {
 			return apierrors.NewBadRequest(fmt.Sprintf("invalid continue token: %v", err))
 		}
 
-		if len(resourceVersion) > 0 && resourceVersion != "0" {
+		if len(opts.ResourceVersion) > 0 && opts.ResourceVersion != "0" {
 			return apierrors.NewBadRequest("specifying resource version is not allowed when using continue")
 		}
-
-		rangeEnd := clientv3.GetPrefixRangeEnd(keyPrefix)
-		options = append(options, clientv3.WithRange(rangeEnd))
 		preparedKey = continueKey
-
 		// If continueRV > 0, the LIST request needs a specific resource version.
 		// continueRV==0 is invalid.
 		// If continueRV < 0, the request is for the latest resource version.
 		if continueRV > 0 {
 			withRev = continueRV
-			returnedRV = continueRV
 		}
-	case recursive && s.pagingEnabled && pred.Limit > 0:
-		if fromRV != nil {
-			switch match {
-			case metav1.ResourceVersionMatchNotOlderThan:
-				// The not older than constraint is checked after we get a response from etcd,
-				// and returnedRV is then set to the revision we get from the etcd response.
-			case metav1.ResourceVersionMatchExact:
-				returnedRV = int64(*fromRV)
-				withRev = returnedRV
-			case "": // legacy case
-				if *fromRV > 0 {
-					returnedRV = int64(*fromRV)
-					withRev = returnedRV
-				}
-			default:
-				return fmt.Errorf("unknown ResourceVersionMatch value: %v", match)
+	case len(opts.ResourceVersion) > 0:
+		parsedRV, err := s.versioner.ParseResourceVersion(opts.ResourceVersion)
+		if err != nil {
+			return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
+		}
+		switch opts.ResourceVersionMatch {
+		case metav1.ResourceVersionMatchNotOlderThan:
+			// The not older than constraint is checked after we get a response from etcd,
+			// and returnedRV is then set to the revision we get from the etcd response.
+		case metav1.ResourceVersionMatchExact:
+			withRev = int64(parsedRV)
+		case "": // legacy case
+			if opts.Recursive && opts.Predicate.Limit > 0 && parsedRV > 0 {
+				withRev = int64(parsedRV)
 			}
-		}
-
-		rangeEnd := clientv3.GetPrefixRangeEnd(keyPrefix)
-		options = append(options, clientv3.WithRange(rangeEnd))
-	default:
-		if fromRV != nil {
-			switch match {
-			case metav1.ResourceVersionMatchNotOlderThan:
-				// The not older than constraint is checked after we get a response from etcd,
-				// and returnedRV is then set to the revision we get from the etcd response.
-			case metav1.ResourceVersionMatchExact:
-				returnedRV = int64(*fromRV)
-				withRev = returnedRV
-			case "": // legacy case
-			default:
-				return fmt.Errorf("unknown ResourceVersionMatch value: %v", match)
-			}
-		}
-
-		if recursive {
-			options = append(options, clientv3.WithPrefix())
+		default:
+			return fmt.Errorf("unknown ResourceVersionMatch value: %v", opts.ResourceVersionMatch)
 		}
 	}
+
 	if withRev != 0 {
 		options = append(options, clientv3.WithRev(withRev))
 	}
@@ -728,7 +691,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 	}()
 
 	metricsOp := "get"
-	if recursive {
+	if opts.Recursive {
 		metricsOp = "list"
 	}
 
@@ -737,10 +700,10 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 		getResp, err = s.client.KV.Get(ctx, preparedKey, options...)
 		metrics.RecordEtcdRequest(metricsOp, s.groupResourceString, err, startTime)
 		if err != nil {
-			return interpretListError(err, len(pred.Continue) > 0, continueKey, keyPrefix)
+			return interpretListError(err, len(opts.Predicate.Continue) > 0, continueKey, keyPrefix)
 		}
 		numFetched += len(getResp.Kvs)
-		if err = s.validateMinimumResourceVersion(resourceVersion, uint64(getResp.Header.Revision)); err != nil {
+		if err = s.validateMinimumResourceVersion(opts.ResourceVersion, uint64(getResp.Header.Revision)); err != nil {
 			return err
 		}
 		hasMore = getResp.More
@@ -748,10 +711,15 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 		if len(getResp.Kvs) == 0 && getResp.More {
 			return fmt.Errorf("no results were found, but etcd indicated there were more values remaining")
 		}
+		// indicate to the client which resource version was returned, and use the same resource version for subsequent requests.
+		if withRev == 0 {
+			withRev = getResp.Header.Revision
+			options = append(options, clientv3.WithRev(withRev))
+		}
 
 		// avoid small allocations for the result slice, since this can be called in many
 		// different contexts and we don't know how significantly the result will be filtered
-		if pred.Empty() {
+		if opts.Predicate.Empty() {
 			growSlice(v, len(getResp.Kvs))
 		} else {
 			growSlice(v, 2048, len(getResp.Kvs))
@@ -759,7 +727,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 
 		// take items from the response until the bucket is full, filtering as we go
 		for i, kv := range getResp.Kvs {
-			if paging && int64(v.Len()) >= pred.Limit {
+			if paging && int64(v.Len()) >= opts.Predicate.Limit {
 				hasMore = true
 				break
 			}
@@ -770,7 +738,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 				return storage.NewInternalErrorf("unable to transform key %q: %v", kv.Key, err)
 			}
 
-			if err := appendListItem(v, data, uint64(kv.ModRevision), pred, s.codec, s.versioner, newItemFunc); err != nil {
+			if err := appendListItem(v, data, uint64(kv.ModRevision), opts.Predicate, s.codec, s.versioner, newItemFunc); err != nil {
 				recordDecodeError(s.groupResourceString, string(kv.Key))
 				return err
 			}
@@ -780,17 +748,12 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 			getResp.Kvs[i] = nil
 		}
 
-		// indicate to the client which resource version was returned
-		if returnedRV == 0 {
-			returnedRV = getResp.Header.Revision
-		}
-
 		// no more results remain or we didn't request paging
 		if !hasMore || !paging {
 			break
 		}
 		// we're paging but we have filled our bucket
-		if int64(v.Len()) >= pred.Limit {
+		if int64(v.Len()) >= opts.Predicate.Limit {
 			break
 		}
 
@@ -804,11 +767,8 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 			*limitOption = clientv3.WithLimit(limit)
 		}
 		preparedKey = string(lastKey) + "\x00"
-		if withRev == 0 {
-			withRev = returnedRV
-			options = append(options, clientv3.WithRev(withRev))
-		}
 	}
+
 	if v.IsNil() {
 		// Ensure that we never return a nil Items pointer in the result for consistency.
 		v.Set(reflect.MakeSlice(v.Type(), 0, 0))
@@ -818,7 +778,7 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 	// we never return a key that the client wouldn't be allowed to see
 	if hasMore {
 		// we want to start immediately after the last key
-		next, err := storage.EncodeContinue(string(lastKey)+"\x00", keyPrefix, returnedRV)
+		next, err := storage.EncodeContinue(string(lastKey)+"\x00", keyPrefix, withRev)
 		if err != nil {
 			return err
 		}
@@ -826,17 +786,15 @@ func (s *store) GetList(ctx context.Context, key string, opts storage.ListOption
 		// getResp.Count counts in objects that do not match the pred.
 		// Instead of returning inaccurate count for non-empty selectors, we return nil.
 		// Only set remainingItemCount if the predicate is empty.
-		if utilfeature.DefaultFeatureGate.Enabled(features.RemainingItemCount) {
-			if pred.Empty() {
-				c := int64(getResp.Count - pred.Limit)
-				remainingItemCount = &c
-			}
+		if opts.Predicate.Empty() {
+			c := int64(getResp.Count - opts.Predicate.Limit)
+			remainingItemCount = &c
 		}
-		return s.versioner.UpdateList(listObj, uint64(returnedRV), next, remainingItemCount)
+		return s.versioner.UpdateList(listObj, uint64(withRev), next, remainingItemCount)
 	}
 
 	// no continuation
-	return s.versioner.UpdateList(listObj, uint64(returnedRV), "", nil)
+	return s.versioner.UpdateList(listObj, uint64(withRev), "", nil)
 }
 
 // growSlice takes a slice value and grows its capacity up
@@ -871,18 +829,7 @@ func growSlice(v reflect.Value, maxCapacity int, sizes ...int) {
 }
 
 // Watch implements storage.Interface.Watch.
-// TODO(#115478): In order to graduate the WatchList feature to beta, the etcd3 implementation must/should also support it.
 func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
-	// it is safe to skip SendInitialEvents if the request is backward compatible
-	// see https://github.com/kubernetes/kubernetes/blob/267eb25e60955fe8e438c6311412e7cf7d028acb/staging/src/k8s.io/apiserver/pkg/storage/etcd3/watcher.go#L260
-	compatibility := opts.Predicate.AllowWatchBookmarks == false && (opts.ResourceVersion == "" || opts.ResourceVersion == "0")
-	if opts.SendInitialEvents != nil && !compatibility {
-		return nil, apierrors.NewInvalid(
-			schema.GroupKind{Group: s.groupResource.Group, Kind: s.groupResource.Resource},
-			"",
-			field.ErrorList{field.Forbidden(field.NewPath("sendInitialEvents"), "for watch is unsupported by an etcd cluster")},
-		)
-	}
 	preparedKey, err := s.prepareKey(key)
 	if err != nil {
 		return nil, err
@@ -891,7 +838,7 @@ func (s *store) Watch(ctx context.Context, key string, opts storage.ListOptions)
 	if err != nil {
 		return nil, err
 	}
-	return s.watcher.Watch(s.watchContext(ctx), preparedKey, int64(rev), opts.Recursive, opts.ProgressNotify, s.transformer, opts.Predicate)
+	return s.watcher.Watch(s.watchContext(ctx), preparedKey, int64(rev), opts)
 }
 
 func (s *store) watchContext(ctx context.Context) context.Context {
@@ -905,6 +852,18 @@ func (s *store) watchContext(ctx context.Context) context.Context {
 	return clientv3.WithRequireLeader(ctx)
 }
 
+func (s *store) getCurrentState(ctx context.Context, key string, v reflect.Value, ignoreNotFound bool) func() (*objState, error) {
+	return func() (*objState, error) {
+		startTime := time.Now()
+		getResp, err := s.client.KV.Get(ctx, key)
+		metrics.RecordEtcdRequest("get", s.groupResourceString, err, startTime)
+		if err != nil {
+			return nil, err
+		}
+		return s.getState(ctx, getResp, key, v, ignoreNotFound)
+	}
+}
+
 func (s *store) getState(ctx context.Context, getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) {
 	state := &objState{
 		meta: &storage.ResponseMeta{},

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -18,27 +18,29 @@ package etcd3
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
-	"reflect"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
+	clientv3 "go.etcd.io/etcd/client/v3"
 	grpccodes "google.golang.org/grpc/codes"
 	grpcstatus "google.golang.org/grpc/status"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apimachinery/pkg/watch"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/value"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
-
-	clientv3 "go.etcd.io/etcd/client/v3"
-
 	"k8s.io/klog/v2"
 )
 
@@ -48,6 +50,9 @@ const (
 	outgoingBufSize = 100
 )
 
+// defaultWatcherMaxLimit is used to facilitate construction tests
+var defaultWatcherMaxLimit int64 = maxLimit
+
 // fatalOnDecodeError is used during testing to panic the server if watcher encounters a decoding error
 var fatalOnDecodeError = false
 
@@ -63,18 +68,19 @@ func TestOnlySetFatalOnDecodeError(b bool) {
 }
 
 type watcher struct {
-	client        *clientv3.Client
-	codec         runtime.Codec
-	newFunc       func() runtime.Object
-	objectType    string
-	groupResource schema.GroupResource
-	versioner     storage.Versioner
+	client              *clientv3.Client
+	codec               runtime.Codec
+	newFunc             func() runtime.Object
+	objectType          string
+	groupResource       schema.GroupResource
+	versioner           storage.Versioner
+	transformer         value.Transformer
+	getCurrentStorageRV func(context.Context) (uint64, error)
 }
 
 // watchChan implements watch.Interface.
 type watchChan struct {
 	watcher           *watcher
-	transformer       value.Transformer
 	key               string
 	initialRev        int64
 	recursive         bool
@@ -87,35 +93,26 @@ type watchChan struct {
 	errChan           chan error
 }
 
-func newWatcher(client *clientv3.Client, codec runtime.Codec, groupResource schema.GroupResource, newFunc func() runtime.Object, versioner storage.Versioner) *watcher {
-	res := &watcher{
-		client:        client,
-		codec:         codec,
-		groupResource: groupResource,
-		newFunc:       newFunc,
-		versioner:     versioner,
-	}
-	if newFunc == nil {
-		res.objectType = "<unknown>"
-	} else {
-		res.objectType = reflect.TypeOf(newFunc()).String()
-	}
-	return res
-}
-
 // Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
 // If rev is zero, it will return the existing object(s) and then start watching from
 // the maximum revision+1 from returned objects.
 // If rev is non-zero, it will watch events happened after given revision.
-// If recursive is false, it watches on given key.
-// If recursive is true, it watches any children and directories under the key, excluding the root key itself.
-// pred must be non-nil. Only if pred matches the change, it will be returned.
-func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive, progressNotify bool, transformer value.Transformer, pred storage.SelectionPredicate) (watch.Interface, error) {
-	if recursive && !strings.HasSuffix(key, "/") {
+// If opts.Recursive is false, it watches on given key.
+// If opts.Recursive is true, it watches any children and directories under the key, excluding the root key itself.
+// pred must be non-nil. Only if opts.Predicate matches the change, it will be returned.
+func (w *watcher) Watch(ctx context.Context, key string, rev int64, opts storage.ListOptions) (watch.Interface, error) {
+	if opts.Recursive && !strings.HasSuffix(key, "/") {
 		key += "/"
 	}
-	wc := w.createWatchChan(ctx, key, rev, recursive, progressNotify, transformer, pred)
-	go wc.run()
+	if opts.ProgressNotify && w.newFunc == nil {
+		return nil, apierrors.NewInternalError(errors.New("progressNotify for watch is unsupported by the etcd storage because no newFunc was provided"))
+	}
+	startWatchRV, err := w.getStartWatchResourceVersion(ctx, rev, opts)
+	if err != nil {
+		return nil, err
+	}
+	wc := w.createWatchChan(ctx, key, startWatchRV, opts.Recursive, opts.ProgressNotify, opts.Predicate)
+	go wc.run(isInitialEventsEndBookmarkRequired(opts), areInitialEventsRequired(rev, opts))
 
 	// For etcd watch we don't have an easy way to answer whether the watch
 	// has already caught up. So in the initial version (given that watchcache
@@ -127,10 +124,9 @@ func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive, p
 	return wc, nil
 }
 
-func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, transformer value.Transformer, pred storage.SelectionPredicate) *watchChan {
+func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan {
 	wc := &watchChan{
 		watcher:           w,
-		transformer:       transformer,
 		key:               key,
 		initialRev:        rev,
 		recursive:         recursive,
@@ -148,6 +144,62 @@ func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, re
 	return wc
 }
 
+// getStartWatchResourceVersion returns a ResourceVersion
+// the watch will be started from.
+// Depending on the input parameters the semantics of the returned ResourceVersion are:
+//   - start at Exact (return resourceVersion)
+//   - start at Most Recent (return an RV from etcd)
+func (w *watcher) getStartWatchResourceVersion(ctx context.Context, resourceVersion int64, opts storage.ListOptions) (int64, error) {
+	if resourceVersion > 0 {
+		return resourceVersion, nil
+	}
+	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
+		return 0, nil
+	}
+	if opts.SendInitialEvents == nil || *opts.SendInitialEvents {
+		// note that when opts.SendInitialEvents=true
+		// we will be issuing a consistent LIST request
+		// against etcd followed by the special bookmark event
+		return 0, nil
+	}
+	// at this point the clients is interested
+	// only in getting a stream of events
+	// starting at the MostRecent point in time (RV)
+	currentStorageRV, err := w.getCurrentStorageRV(ctx)
+	if err != nil {
+		return 0, err
+	}
+	// currentStorageRV is taken from resp.Header.Revision (int64)
+	// and cast to uint64, so it is safe to do reverse
+	// at some point we should unify the interface but that
+	// would require changing  Versioner.UpdateList
+	return int64(currentStorageRV), nil
+}
+
+// isInitialEventsEndBookmarkRequired since there is no way to directly set
+// opts.ProgressNotify from the API and the etcd3 impl doesn't support
+// notification for external clients we simply return initialEventsEndBookmarkRequired
+// to only send the bookmark event after the initial list call.
+//
+// see: https://github.com/kubernetes/kubernetes/issues/120348
+func isInitialEventsEndBookmarkRequired(opts storage.ListOptions) bool {
+	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
+		return false
+	}
+	return opts.SendInitialEvents != nil && *opts.SendInitialEvents && opts.Predicate.AllowWatchBookmarks
+}
+
+// areInitialEventsRequired returns true if all events from the etcd should be returned.
+func areInitialEventsRequired(resourceVersion int64, opts storage.ListOptions) bool {
+	if opts.SendInitialEvents == nil && resourceVersion == 0 {
+		return true // legacy case
+	}
+	if !utilfeature.DefaultFeatureGate.Enabled(features.WatchList) {
+		return false
+	}
+	return opts.SendInitialEvents != nil && *opts.SendInitialEvents
+}
+
 type etcdError interface {
 	Code() grpccodes.Code
 	Error() string
@@ -173,9 +225,9 @@ func isCancelError(err error) bool {
 	return false
 }
 
-func (wc *watchChan) run() {
+func (wc *watchChan) run(initialEventsEndBookmarkRequired, forceInitialEvents bool) {
 	watchClosedCh := make(chan struct{})
-	go wc.startWatching(watchClosedCh)
+	go wc.startWatching(watchClosedCh, initialEventsEndBookmarkRequired, forceInitialEvents)
 
 	var resultChanWG sync.WaitGroup
 	resultChanWG.Add(1)
@@ -225,17 +277,58 @@ func (wc *watchChan) RequestWatchProgress() error {
 func (wc *watchChan) sync() error {
 	opts := []clientv3.OpOption{}
 	if wc.recursive {
-		opts = append(opts, clientv3.WithPrefix())
+		opts = append(opts, clientv3.WithLimit(defaultWatcherMaxLimit))
+		rangeEnd := clientv3.GetPrefixRangeEnd(wc.key)
+		opts = append(opts, clientv3.WithRange(rangeEnd))
 	}
-	getResp, err := wc.watcher.client.Get(wc.ctx, wc.key, opts...)
-	if err != nil {
-		return err
+
+	var err error
+	var lastKey []byte
+	var withRev int64
+	var getResp *clientv3.GetResponse
+
+	metricsOp := "get"
+	if wc.recursive {
+		metricsOp = "list"
 	}
-	wc.initialRev = getResp.Header.Revision
-	for _, kv := range getResp.Kvs {
-		wc.sendEvent(parseKV(kv))
+
+	preparedKey := wc.key
+
+	for {
+		startTime := time.Now()
+		getResp, err = wc.watcher.client.KV.Get(wc.ctx, preparedKey, opts...)
+		metrics.RecordEtcdRequest(metricsOp, wc.watcher.groupResource.String(), err, startTime)
+		if err != nil {
+			return interpretListError(err, true, preparedKey, wc.key)
+		}
+
+		if len(getResp.Kvs) == 0 && getResp.More {
+			return fmt.Errorf("no results were found, but etcd indicated there were more values remaining")
+		}
+
+		// send items from the response until no more results
+		for i, kv := range getResp.Kvs {
+			lastKey = kv.Key
+			wc.sendEvent(parseKV(kv))
+			// free kv early. Long lists can take O(seconds) to decode.
+			getResp.Kvs[i] = nil
+		}
+
+		if withRev == 0 {
+			wc.initialRev = getResp.Header.Revision
+		}
+
+		// no more results remain
+		if !getResp.More {
+			return nil
+		}
+
+		preparedKey = string(lastKey) + "\x00"
+		if withRev == 0 {
+			withRev = getResp.Header.Revision
+			opts = append(opts, clientv3.WithRev(withRev))
+		}
 	}
-	return nil
 }
 
 func logWatchChannelErr(err error) {
@@ -253,14 +346,44 @@ func logWatchChannelErr(err error) {
 // startWatching does:
 // - get current objects if initialRev=0; set initialRev to current rev
 // - watch on given key and send events to process.
-func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
-	if wc.initialRev == 0 {
+//
+// initialEventsEndBookmarkSent helps us keep track
+// of whether we have sent an annotated bookmark event.
+//
+// it's important to note that we don't
+// need to track the actual RV because
+// we only send the bookmark event
+// after the initial list call.
+//
+// when this variable is set to false,
+// it means we don't have any specific
+// preferences for delivering bookmark events.
+func (wc *watchChan) startWatching(watchClosedCh chan struct{}, initialEventsEndBookmarkRequired, forceInitialEvents bool) {
+	if wc.initialRev > 0 && forceInitialEvents {
+		currentStorageRV, err := wc.watcher.getCurrentStorageRV(wc.ctx)
+		if err != nil {
+			wc.sendError(err)
+			return
+		}
+		if uint64(wc.initialRev) > currentStorageRV {
+			wc.sendError(storage.NewTooLargeResourceVersionError(uint64(wc.initialRev), currentStorageRV, int(wait.Jitter(1*time.Second, 3).Seconds())))
+			return
+		}
+	}
+	if forceInitialEvents {
 		if err := wc.sync(); err != nil {
 			klog.Errorf("failed to sync with latest state: %v", err)
 			wc.sendError(err)
 			return
 		}
 	}
+	if initialEventsEndBookmarkRequired {
+		wc.sendEvent(func() *event {
+			e := progressNotifyEvent(wc.initialRev)
+			e.isInitialEventsEndBookmark = true
+			return e
+		}())
+	}
 	opts := []clientv3.OpOption{clientv3.WithRev(wc.initialRev + 1), clientv3.WithPrevKV()}
 	if wc.recursive {
 		opts = append(opts, clientv3.WithPrefix())
@@ -352,14 +475,17 @@ func (wc *watchChan) transform(e *event) (res *watch.Event) {
 
 	switch {
 	case e.isProgressNotify:
-		if wc.watcher.newFunc == nil {
-			return nil
-		}
 		object := wc.watcher.newFunc()
 		if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil {
 			klog.Errorf("failed to propagate object version: %v", err)
 			return nil
 		}
+		if e.isInitialEventsEndBookmark {
+			if err := storage.AnnotateInitialEventsEndBookmark(object); err != nil {
+				wc.sendError(fmt.Errorf("error while accessing object's metadata gr: %v, type: %v, obj: %#v, err: %v", wc.watcher.groupResource, wc.watcher.objectType, object, err))
+				return nil
+			}
+		}
 		res = &watch.Event{
 			Type:   watch.Bookmark,
 			Object: object,
@@ -447,7 +573,7 @@ func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtim
 	}
 
 	if !e.isDeleted {
-		data, _, err := wc.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key))
+		data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.value, authenticatedDataString(e.key))
 		if err != nil {
 			return nil, nil, err
 		}
@@ -462,7 +588,7 @@ func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtim
 	// we need the object only to compute whether it was filtered out
 	// before).
 	if len(e.prevValue) > 0 && (e.isDeleted || !wc.acceptAll()) {
-		data, _, err := wc.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key))
+		data, _, err := wc.watcher.transformer.TransformFromStorage(wc.ctx, e.prevValue, authenticatedDataString(e.key))
 		if err != nil {
 			return nil, nil, err
 		}

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -282,6 +282,19 @@ type ListOptions struct {
 	Recursive bool
 	// ProgressNotify determines whether storage-originated bookmark (progress notify) events should
 	// be delivered to the users. The option is ignored for non-watch requests.
+	//
+	// Firstly, note that this field is different from the Predicate.AllowWatchBookmarks field.
+	// Secondly, this field is intended for internal clients only such as the watch cache.
+	//
+	// This means that external clients do not have the ability to set this field directly.
+	// For example by setting the allowWatchBookmarks query parameter.
+	//
+	// The motivation for this approach is the fact that the frequency
+	// of bookmark events from a storage like etcd might be very high.
+	// As the number of watch requests increases, the server load would also increase.
+	//
+	// Furthermore, the server is not obligated to provide bookmark events at all,
+	// as described in https://github.com/kubernetes/enhancements/tree/master/keps/sig-api-machinery/956-watch-bookmark#proposal
 	ProgressNotify bool
 	// SendInitialEvents, when set together with Watch option,
 	// begin the watch stream with synthetic init events to build the

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -62,11 +62,6 @@ type Config struct {
 	Prefix string
 	// Transport holds all connection related info, i.e. equal TransportConfig means equal servers we talk to.
 	Transport TransportConfig
-	// Paging indicates whether the server implementation should allow paging (if it is
-	// supported). This is generally configured by feature gating, or by a specific
-	// resource type not wishing to allow paging, and is not intended for end users to
-	// set.
-	Paging bool
 
 	Codec runtime.Codec
 	// EncodeVersioner is the same groupVersioner used to build the
@@ -115,7 +110,6 @@ func (config *Config) ForResource(resource schema.GroupResource) *ConfigForResou
 
 func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
 	return &Config{
-		Paging:               true,
 		Prefix:               prefix,
 		Codec:                codec,
 		CompactionInterval:   DefaultCompactInterval,

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -419,7 +419,7 @@ func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration
 	}, nil
 }
 
-func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
+func newETCD3Storage(c storagebackend.ConfigForResource, newFunc, newListFunc func() runtime.Object, resourcePrefix string) (storage.Interface, DestroyFunc, error) {
 	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
 	if err != nil {
 		return nil, nil, err
@@ -454,7 +454,7 @@ func newETCD3Storage(c storagebackend.ConfigForResource, newFunc func() runtime.
 	if transformer == nil {
 		transformer = identity.NewEncryptCheckTransformer()
 	}
-	return etcd3.New(client, c.Codec, newFunc, c.Prefix, c.GroupResource, transformer, c.Paging, c.LeaseManagerConfig), destroyFunc, nil
+	return etcd3.New(client, c.Codec, newFunc, newListFunc, c.Prefix, resourcePrefix, c.GroupResource, transformer, c.LeaseManagerConfig), destroyFunc, nil
 }
 
 // startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go

@@ -30,12 +30,12 @@ import (
 type DestroyFunc func()
 
 // Create creates a storage backend based on given config.
-func Create(c storagebackend.ConfigForResource, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
+func Create(c storagebackend.ConfigForResource, newFunc, newListFunc func() runtime.Object, resourcePrefix string) (storage.Interface, DestroyFunc, error) {
 	switch c.Type {
 	case storagebackend.StorageTypeETCD2:
 		return nil, nil, fmt.Errorf("%s is no longer a supported storage backend", c.Type)
 	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
-		return newETCD3Storage(c, newFunc)
+		return newETCD3Storage(c, newFunc, newListFunc, resourcePrefix)
 	default:
 		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
 	}

vendor/k8s.io/apiserver/pkg/storage/util.go

@@ -17,14 +17,25 @@ limitations under the License.
 package storage
 
 import (
+	"context"
 	"fmt"
+	"strconv"
 	"sync/atomic"
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/validation/path"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
+const (
+	// initialEventsAnnotationKey the name of the key
+	// under which an annotation marking the end of list stream
+	// is kept.
+	initialEventsAnnotationKey = "k8s.io/initial-events-end"
+)
+
 type SimpleUpdateFunc func(runtime.Object) (runtime.Object, error)
 
 // SimpleUpdateFunc converts SimpleUpdateFunc into UpdateFunc
@@ -79,3 +90,72 @@ func (hwm *HighWaterMark) Update(current int64) bool {
 		}
 	}
 }
+
+// GetCurrentResourceVersionFromStorage gets the current resource version from the underlying storage engine.
+// This method issues an empty list request and reads only the ResourceVersion from the object metadata
+func GetCurrentResourceVersionFromStorage(ctx context.Context, storage Interface, newListFunc func() runtime.Object, resourcePrefix, objectType string) (uint64, error) {
+	if storage == nil {
+		return 0, fmt.Errorf("storage wasn't provided for %s", objectType)
+	}
+	if newListFunc == nil {
+		return 0, fmt.Errorf("newListFunction wasn't provided for %s", objectType)
+	}
+	emptyList := newListFunc()
+	pred := SelectionPredicate{
+		Label: labels.Everything(),
+		Field: fields.Everything(),
+		Limit: 1, // just in case we actually hit something
+	}
+
+	err := storage.GetList(ctx, resourcePrefix, ListOptions{Predicate: pred}, emptyList)
+	if err != nil {
+		return 0, err
+	}
+	emptyListAccessor, err := meta.ListAccessor(emptyList)
+	if err != nil {
+		return 0, err
+	}
+	if emptyListAccessor == nil {
+		return 0, fmt.Errorf("unable to extract a list accessor from %T", emptyList)
+	}
+
+	currentResourceVersion, err := strconv.Atoi(emptyListAccessor.GetResourceVersion())
+	if err != nil {
+		return 0, err
+	}
+
+	if currentResourceVersion == 0 {
+		return 0, fmt.Errorf("the current resource version must be greater than 0")
+	}
+	return uint64(currentResourceVersion), nil
+}
+
+// AnnotateInitialEventsEndBookmark adds a special annotation to the given object
+// which indicates that the initial events have been sent.
+//
+// Note that this function assumes that the obj's annotation
+// field is a reference type (i.e. a map).
+func AnnotateInitialEventsEndBookmark(obj runtime.Object) error {
+	objMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return err
+	}
+	objAnnotations := objMeta.GetAnnotations()
+	if objAnnotations == nil {
+		objAnnotations = map[string]string{}
+	}
+	objAnnotations[initialEventsAnnotationKey] = "true"
+	objMeta.SetAnnotations(objAnnotations)
+	return nil
+}
+
+// HasInitialEventsEndBookmarkAnnotation checks the presence of the
+// special annotation which marks that the initial events have been sent.
+func HasInitialEventsEndBookmarkAnnotation(obj runtime.Object) (bool, error) {
+	objMeta, err := meta.Accessor(obj)
+	if err != nil {
+		return false, err
+	}
+	objAnnotations := objMeta.GetAnnotations()
+	return objAnnotations[initialEventsAnnotationKey] == "true", nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/cache.go

@@ -26,6 +26,7 @@ import (
 
 	utilcache "k8s.io/apimachinery/pkg/util/cache"
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	"k8s.io/utils/clock"
 )
 
@@ -38,10 +39,13 @@ type simpleCache struct {
 	ttl   time.Duration
 	// hashPool is a per cache pool of hash.Hash (to avoid allocations from building the Hash)
 	// SHA-256 is used to prevent collisions
-	hashPool *sync.Pool
+	hashPool        *sync.Pool
+	providerName    string
+	mu              sync.Mutex                          // guards call to set
+	recordCacheSize func(providerName string, size int) // for unit tests
 }
 
-func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
+func newSimpleCache(clock clock.Clock, ttl time.Duration, providerName string) *simpleCache {
 	cache := utilcache.NewExpiringWithClock(clock)
 	cache.AllowExpiredGet = true // for a given key, the value (the decryptTransformer) is always the same
 	return &simpleCache{
@@ -52,6 +56,8 @@ func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
 				return sha256.New()
 			},
 		},
+		providerName:    providerName,
+		recordCacheSize: metrics.RecordDekSourceCacheSize,
 	}
 }
 
@@ -66,6 +72,8 @@ func (c *simpleCache) get(key []byte) value.Read {
 
 // set caches the record for the key
 func (c *simpleCache) set(key []byte, transformer value.Read) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
 	if len(key) == 0 {
 		panic("key must not be empty")
 	}
@@ -73,6 +81,8 @@ func (c *simpleCache) set(key []byte, transformer value.Read) {
 		panic("transformer must not be nil")
 	}
 	c.cache.Set(c.keyFunc(key), transformer, c.ttl)
+	// Add metrics for cache size
+	c.recordCacheSize(c.providerName, c.cache.Len())
 }
 
 // keyFunc generates a string key by hashing the inputs.

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -28,6 +28,7 @@ import (
 	"unsafe"
 
 	"github.com/gogo/protobuf/proto"
+	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/crypto/cryptobyte"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -39,21 +40,22 @@ import (
 	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
 	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
+	"k8s.io/component-base/tracing"
 	"k8s.io/klog/v2"
 	kmsservice "k8s.io/kms/pkg/service"
 	"k8s.io/utils/clock"
 )
 
-// TODO integration test with old AES GCM data recorded and new KDF data recorded
-
 func init() {
 	value.RegisterMetrics()
 	metrics.RegisterMetrics()
 }
 
 const (
-	// KMSAPIVersion is the version of the KMS API.
-	KMSAPIVersion = "v2beta1"
+	// KMSAPIVersionv2 is a version of the KMS API.
+	KMSAPIVersionv2 = "v2"
+	// KMSAPIVersionv2beta1 is a version of the KMS API.
+	KMSAPIVersionv2beta1 = "v2beta1"
 	// annotationsMaxSize is the maximum size of the annotations.
 	annotationsMaxSize = 32 * 1024 // 32 kB
 	// KeyIDMaxSize is the maximum size of the keyID.
@@ -112,32 +114,51 @@ type envelopeTransformer struct {
 	stateFunc       StateFunc
 
 	// cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form.
-	cache *simpleCache
+	cache       *simpleCache
+	apiServerID string
 }
 
 // NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
 // the data items they encrypt.
-func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc) value.Transformer {
-	return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, cacheTTL, clock.RealClock{})
+func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string) value.Transformer {
+	return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, apiServerID, cacheTTL, clock.RealClock{})
 }
 
-func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
+func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, apiServerID string, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
 	return &envelopeTransformer{
 		envelopeService: envelopeService,
 		providerName:    providerName,
 		stateFunc:       stateFunc,
-		cache:           newSimpleCache(clock, cacheTTL),
+		cache:           newSimpleCache(clock, cacheTTL, providerName),
+		apiServerID:     apiServerID,
 	}
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
 func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
+	ctx, span := tracing.Start(ctx, "TransformFromStorage with envelopeTransformer",
+		attribute.String("transformer.provider.name", t.providerName),
+		// The service.instance_id of the apiserver is already available in the trace
+		/*
+			{
+			"key": "service.instance.id",
+			"type": "string",
+			"value": "apiserver-zsteyir5lyrtdcmqqmd5kzze6m"
+			}
+		*/
+	)
+	defer span.End(500 * time.Millisecond)
+
+	span.AddEvent("About to decode encrypted object")
 	// Deserialize the EncryptedObject from the data.
 	encryptedObject, err := t.doDecode(data)
 	if err != nil {
+		span.AddEvent("Decoding encrypted object failed")
+		span.RecordError(err)
 		return nil, false, err
 	}
+	span.AddEvent("Decoded encrypted object")
 
 	useSeed := encryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
 
@@ -158,6 +179,7 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 
 	// fallback to the envelope service if we do not have the transformer locally
 	if transformer == nil {
+		span.AddEvent("About to decrypt DEK using remote service")
 		value.RecordCacheMiss()
 
 		requestInfo := getRequestInfoFromContext(ctx)
@@ -172,21 +194,28 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 			Annotations: encryptedObject.Annotations,
 		})
 		if err != nil {
+			span.AddEvent("DEK decryption failed")
+			span.RecordError(err)
 			return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
 		}
+		span.AddEvent("DEK decryption succeeded")
 
 		transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key, useSeed)
 		if err != nil {
 			return nil, false, err
 		}
 	}
-	metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID)
+	metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID, t.apiServerID)
 
+	span.AddEvent("About to decrypt data using DEK")
 	out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
 	if err != nil {
+		span.AddEvent("Data decryption failed")
+		span.RecordError(err)
 		return nil, false, err
 	}
 
+	span.AddEvent("Data decryption succeeded")
 	// data is considered stale if the key ID does not match our current write transformer
 	return out,
 		stale ||
@@ -197,6 +226,19 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 
 // TransformToStorage encrypts data to be written to disk using envelope encryption.
 func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
+	ctx, span := tracing.Start(ctx, "TransformToStorage with envelopeTransformer",
+		attribute.String("transformer.provider.name", t.providerName),
+		// The service.instance_id of the apiserver is already available in the trace
+		/*
+			{
+			"key": "service.instance.id",
+			"type": "string",
+			"value": "apiserver-zsteyir5lyrtdcmqqmd5kzze6m"
+			}
+		*/
+	)
+	defer span.End(500 * time.Millisecond)
+
 	state, err := t.stateFunc()
 	if err != nil {
 		return nil, err
@@ -208,7 +250,6 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 	// this prevents a cache miss every time the DEK rotates
 	// this has the side benefit of causing the cache to perform a GC
 	// TODO see if we can do this inside the stateFunc control loop
-	// TODO(aramase): Add metrics for cache size.
 	t.cache.set(state.CacheKey, state.Transformer)
 
 	requestInfo := getRequestInfoFromContext(ctx)
@@ -216,18 +257,31 @@ func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byt
 		"group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource,
 		"verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
 
+	span.AddEvent("About to encrypt data using DEK")
 	result, err := state.Transformer.TransformToStorage(ctx, data, dataCtx)
 	if err != nil {
+		span.AddEvent("Data encryption failed")
+		span.RecordError(err)
 		return nil, err
 	}
+	span.AddEvent("Data encryption succeeded")
 
-	metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID)
+	metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.EncryptedObject.KeyID, t.apiServerID)
 
 	encObjectCopy := state.EncryptedObject
 	encObjectCopy.EncryptedData = result
 
+	span.AddEvent("About to encode encrypted object")
 	// Serialize the EncryptedObject to a byte array.
-	return t.doEncode(&encObjectCopy)
+	out, err := t.doEncode(&encObjectCopy)
+	if err != nil {
+		span.AddEvent("Encoding encrypted object failed")
+		span.RecordError(err)
+		return nil, err
+	}
+	span.AddEvent("Encoded encrypted object")
+
+	return out, nil
 }
 
 // addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
@@ -250,7 +304,6 @@ func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key [
 	if err != nil {
 		return nil, err
 	}
-	// TODO(aramase): Add metrics for cache size.
 	t.cache.set(cacheKey, transformer)
 	return transformer, nil
 }

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.pb.go

@@ -71,11 +71,20 @@ type EncryptedObject struct {
 	// EncryptedData is the encrypted data.
 	EncryptedData []byte `protobuf:"bytes,1,opt,name=encryptedData,proto3" json:"encryptedData,omitempty"`
 	// KeyID is the KMS key ID used for encryption operations.
+	// keyID must satisfy the following constraints:
+	// 1. The keyID is not empty.
+	// 2. The size of keyID is less than 1 kB.
 	KeyID string `protobuf:"bytes,2,opt,name=keyID,proto3" json:"keyID,omitempty"`
 	// EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
 	// encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
+	// encryptedDEKSource must satisfy the following constraints:
+	// 1. The encrypted DEK source is not empty.
+	// 2. The size of encrypted DEK source is less than 1 kB.
 	EncryptedDEKSource []byte `protobuf:"bytes,3,opt,name=encryptedDEKSource,proto3" json:"encryptedDEKSource,omitempty"`
 	// Annotations is additional metadata that was provided by the KMS plugin.
+	// Annotations must satisfy the following constraints:
+	//  1. Annotation key must be a fully qualified domain name that conforms to the definition in DNS (RFC 1123).
+	//  2. The size of annotations keys + values is less than 32 kB.
 	Annotations map[string][]byte `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
 	// encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.
 	EncryptedDEKSourceType EncryptedDEKSourceType `protobuf:"varint,5,opt,name=encryptedDEKSourceType,proto3,enum=v2.EncryptedDEKSourceType" json:"encryptedDEKSourceType,omitempty"`

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.proto

@@ -26,13 +26,22 @@ message EncryptedObject {
   bytes encryptedData = 1;
 
   // KeyID is the KMS key ID used for encryption operations.
+  // keyID must satisfy the following constraints:
+  // 1. The keyID is not empty.
+  // 2. The size of keyID is less than 1 kB.
   string keyID = 2;
 
   // EncryptedDEKSource is the ciphertext of the source of the DEK used to encrypt the data stored in encryptedData.
   // encryptedDEKSourceType defines the process of using the plaintext of this field to determine the aforementioned DEK.
+  // encryptedDEKSource must satisfy the following constraints:
+  // 1. The encrypted DEK source is not empty.
+  // 2. The size of encrypted DEK source is less than 1 kB.
   bytes encryptedDEKSource = 3;
 
   // Annotations is additional metadata that was provided by the KMS plugin.
+  // Annotations must satisfy the following constraints:
+  //  1. Annotation key must be a fully qualified domain name that conforms to the definition in DNS (RFC 1123).
+  //  2. The size of annotations keys + values is less than 32 kB.
   map<string, bytes> annotations = 4;
 
   // encryptedDEKSourceType defines the process of using the plaintext of encryptedDEKSource to determine the DEK.

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go

@@ -44,6 +44,7 @@ type metricLabels struct {
 	transformationType string
 	providerName       string
 	keyIDHash          string
+	apiServerIDHash    string
 }
 
 /*
@@ -107,21 +108,21 @@ var (
 
 	// keyIDHashTotal is the number of times a keyID is used
 	// e.g. apiserver_envelope_encryption_key_id_hash_total counter
-	// apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256",
+	// apiserver_envelope_encryption_key_id_hash_total{apiserver_id_hash="sha256",key_id_hash="sha256",
 	// provider_name="providerName",transformation_type="from_storage"} 1
 	KeyIDHashTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "key_id_hash_total",
-			Help:           "Number of times a keyID is used split by transformation type and provider.",
+			Help:           "Number of times a keyID is used split by transformation type, provider, and apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"transformation_type", "provider_name", "key_id_hash"},
+		[]string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
 	)
 
 	// keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used
-	// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
+	// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
 	KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec(
 		&metrics.GaugeOpts{
 			Namespace:      namespace,
@@ -130,11 +131,11 @@ var (
 			Help:           "The last time in seconds when a keyID was used.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"transformation_type", "provider_name", "key_id_hash"},
+		[]string{"transformation_type", "provider_name", "key_id_hash", "apiserver_id_hash"},
 	)
 
 	// keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call.
-	// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
+	// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{apiserver_id_hash="sha256",key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
 	KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec(
 		&metrics.GaugeOpts{
 			Namespace:      namespace,
@@ -143,7 +144,7 @@ var (
 			Help:           "The last time in seconds when a keyID was returned by the Status RPC call.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"provider_name", "key_id_hash"},
+		[]string{"provider_name", "key_id_hash", "apiserver_id_hash"},
 	)
 
 	InvalidKeyIDFromStatusTotal = metrics.NewCounterVec(
@@ -156,6 +157,17 @@ var (
 		},
 		[]string{"provider_name", "error"},
 	)
+
+	DekSourceCacheSize = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dek_source_cache_size",
+			Help:           "Number of records in data encryption key (DEK) source cache. On a restart, this value is an approximation of the number of decrypt RPC calls the server will make to the KMS plugin.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"provider_name"},
+	)
 )
 
 var registerMetricsFunc sync.Once
@@ -171,19 +183,19 @@ func registerLRUMetrics() {
 
 	keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
 		item := key.(metricLabels)
-		if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
+		if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
 			klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType,
-				"providerName", item.providerName, "keyIDHash", item.keyIDHash)
+				"providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
 		}
-		if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
+		if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
 			klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType,
-				"providerName", item.providerName, "keyIDHash", item.keyIDHash)
+				"providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
 		}
 	})
 	keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
 		item := key.(metricLabels)
-		if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash); deleted {
-			klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash)
+		if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash, item.apiServerIDHash); deleted {
+			klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash, "apiServerIDHash", item.apiServerIDHash)
 		}
 	})
 }
@@ -197,6 +209,7 @@ func RegisterMetrics() {
 		}
 		legacyregistry.MustRegister(dekCacheFillPercent)
 		legacyregistry.MustRegister(dekCacheInterArrivals)
+		legacyregistry.MustRegister(DekSourceCacheSize)
 		legacyregistry.MustRegister(KeyIDHashTotal)
 		legacyregistry.MustRegister(KeyIDHashLastTimestampSeconds)
 		legacyregistry.MustRegister(KeyIDHashStatusLastTimestampSeconds)
@@ -206,22 +219,22 @@ func RegisterMetrics() {
 }
 
 // RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations
-func RecordKeyID(transformationType, providerName, keyID string) {
+func RecordKeyID(transformationType, providerName, keyID, apiServerID string) {
 	lockRecordKeyID.Lock()
 	defer lockRecordKeyID.Unlock()
 
-	keyIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID)
-	KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash).Inc()
-	KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash).SetToCurrentTime()
+	keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID, apiServerID)
+	KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).Inc()
+	KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
 }
 
 // RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call.
-func RecordKeyIDFromStatus(providerName, keyID string) {
+func RecordKeyIDFromStatus(providerName, keyID, apiServerID string) {
 	lockRecordKeyIDStatus.Lock()
 	defer lockRecordKeyIDStatus.Unlock()
 
-	keyIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID)
-	KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash).SetToCurrentTime()
+	keyIDHash, apiServerIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID, apiServerID)
+	KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash, apiServerIDHash).SetToCurrentTime()
 }
 
 func RecordInvalidKeyIDFromStatus(providerName, errCode string) {
@@ -255,6 +268,10 @@ func RecordDekCacheFillPercent(percent float64) {
 	dekCacheFillPercent.Set(percent)
 }
 
+func RecordDekSourceCacheSize(providerName string, size int) {
+	DekSourceCacheSize.WithLabelValues(providerName).Set(float64(size))
+}
+
 // RecordKMSOperationLatency records the latency of KMS operation.
 func RecordKMSOperationLatency(providerName, methodName string, duration time.Duration, err error) {
 	KMSOperationsLatencyMetric.WithLabelValues(providerName, methodName, getErrorCode(err)).Observe(duration.Seconds())
@@ -281,24 +298,25 @@ func getErrorCode(err error) string {
 }
 
 func getHash(data string) string {
+	if len(data) == 0 {
+		return ""
+	}
 	h := hashPool.Get().(hash.Hash)
 	h.Reset()
 	h.Write([]byte(data))
-	result := fmt.Sprintf("sha256:%x", h.Sum(nil))
+	dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
 	hashPool.Put(h)
-	return result
+	return dataHash
 }
 
-func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID string) string {
-	keyIDHash := ""
-	// only get hash if the keyID is not empty
-	if len(keyID) > 0 {
-		keyIDHash = getHash(keyID)
-	}
+func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID, apiServerID string) (string, string) {
+	keyIDHash := getHash(keyID)
+	apiServerIDHash := getHash(apiServerID)
 	c.Add(metricLabels{
 		transformationType: transformationType,
 		providerName:       providerName,
 		keyIDHash:          keyIDHash,
+		apiServerIDHash:    apiServerIDHash,
 	}, nil) // value is irrelevant, this is a set and not a map
-	return keyIDHash
+	return keyIDHash, apiServerIDHash
 }

vendor/k8s.io/apiserver/pkg/util/apihelpers/helpers.go

@@ -19,7 +19,7 @@ package apihelpers
 import (
 	"sort"
 
-	flowcontrol "k8s.io/api/flowcontrol/v1beta3"
+	flowcontrol "k8s.io/api/flowcontrol/v1"
 )
 
 // SetFlowSchemaCondition sets conditions.