rebase: update k8s.io packages to v0.29.0

Signed-off-by: Niels de Vos <ndevos@ibm.com>

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -42,6 +42,9 @@ func NewAPIEnablementOptions() *APIEnablementOptions {
 
 // AddFlags adds flags for a specific APIServer to the specified FlagSet
 func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
 	fs.Var(&s.RuntimeConfig, "runtime-config", ""+
 		"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
 		"v1=true|false for the core API group\n"+
@@ -87,7 +90,6 @@ func (s *APIEnablementOptions) Validate(registries ...GroupRegistry) []error {
 
 // ApplyTo override MergedResourceConfig with defaults and registry
 func (s *APIEnablementOptions) ApplyTo(c *server.Config, defaultResourceConfig *serverstore.ResourceConfig, registry resourceconfig.GroupVersionRegistry) error {
-
 	if s == nil {
 		return nil
 	}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -105,10 +105,36 @@ const (
 	kmsReloadHealthCheckName = "kms-providers"
 )
 
+var codecs serializer.CodecFactory
+
+// this atomic bool allows us to swap enablement of the KMSv2KDF feature in tests
+// as the feature gate is now locked to true starting with v1.29
+// Note: it cannot be set by an end user
+var kdfDisabled atomic.Bool
+
+// this function should only be called in tests to swap enablement of the KMSv2KDF feature
+func SetKDFForTests(b bool) func() {
+	kdfDisabled.Store(!b)
+	return func() {
+		kdfDisabled.Store(false)
+	}
+}
+
+// this function should be used to determine enablement of the KMSv2KDF feature
+// instead of getting it from DefaultFeatureGate as the feature gate is now locked
+// to true starting with v1.29
+func GetKDF() bool {
+	return !kdfDisabled.Load()
+}
+
 func init() {
-	metrics.RegisterMetrics()
-	storagevalue.RegisterMetrics()
+	configScheme := runtime.NewScheme()
+	utilruntime.Must(apiserverconfig.AddToScheme(configScheme))
+	utilruntime.Must(apiserverconfigv1.AddToScheme(configScheme))
+	codecs = serializer.NewCodecFactory(configScheme)
 	envelopemetrics.RegisterMetrics()
+	storagevalue.RegisterMetrics()
+	metrics.RegisterMetrics()
 }
 
 type kmsPluginHealthzResponse struct {
@@ -131,6 +157,8 @@ type kmsv2PluginProbe struct {
 	service      kmsservice.Service
 	lastResponse *kmsPluginHealthzResponse
 	l            *sync.Mutex
+	apiServerID  string
+	version      string
 }
 
 type kmsHealthChecker []healthz.HealthChecker
@@ -184,13 +212,13 @@ type EncryptionConfiguration struct {
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
 // If reload is true, or KMS v2 plugins are used with no KMS v1 plugins, the returned slice of health checkers will always be of length 1.
-func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*EncryptionConfiguration, error) {
+func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool, apiServerID string) (*EncryptionConfiguration, error) {
 	config, contentHash, err := loadConfig(filepath, reload)
 	if err != nil {
 		return nil, fmt.Errorf("error while parsing file: %w", err)
 	}
 
-	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config)
+	transformers, kmsHealthChecks, kmsUsed, err := getTransformerOverridesAndKMSPluginHealthzCheckers(ctx, config, apiServerID)
 	if err != nil {
 		return nil, fmt.Errorf("error while building transformers: %w", err)
 	}
@@ -215,9 +243,9 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
 // getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
 	var kmsHealthChecks []healthz.HealthChecker
-	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
+	transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config, apiServerID)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -236,7 +264,7 @@ type healthChecker interface {
 // getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
+func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration, apiServerID string) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
 	resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -245,7 +273,7 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
 	for _, resourceConfig := range config.Resources {
 		resourceConfig := resourceConfig
 
-		transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig)
+		transformers, p, used, err := prefixTransformersAndProbes(ctx, resourceConfig, apiServerID)
 		if err != nil {
 			return nil, nil, nil, err
 		}
@@ -362,7 +390,7 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
 	// this gate can only change during tests, but the check is cheap enough to always make
 	// this allows us to easily exercise both modes without restarting the API server
 	// TODO integration test that this dynamically takes effect
-	useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
+	useSeed := GetKDF()
 	stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
 
 	// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
@@ -447,15 +475,23 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 	if response.Healthz != "ok" {
 		errs = append(errs, fmt.Errorf("got unexpected healthz status: %s", response.Healthz))
 	}
-	if response.Version != envelopekmsv2.KMSAPIVersion {
-		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersion, response.Version))
+	if response.Version != envelopekmsv2.KMSAPIVersionv2 && response.Version != envelopekmsv2.KMSAPIVersionv2beta1 {
+		errs = append(errs, fmt.Errorf("expected KMSv2 API version %s, got %s", envelopekmsv2.KMSAPIVersionv2, response.Version))
+	} else {
+		// set version for the first status response
+		if len(h.version) == 0 {
+			h.version = response.Version
+		}
+		if h.version != response.Version {
+			errs = append(errs, fmt.Errorf("KMSv2 API version should not change after the initial status response version %s, got %s", h.version, response.Version))
+		}
 	}
 
 	if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
 		envelopemetrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
 		errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
 	} else {
-		envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID)
+		envelopemetrics.RecordKeyIDFromStatus(h.name, response.KeyID, h.apiServerID)
 		// unconditionally append as we filter out nil errors below
 		errs = append(errs, h.rotateDEKOnKeyIDChange(ctx, response.KeyID, string(uuid.NewUUID())))
 	}
@@ -468,6 +504,24 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
 
 // loadConfig parses the encryption configuration file at filepath and returns the parsed config and hash of the file.
 func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfiguration, string, error) {
+	data, contentHash, err := loadDataAndHash(filepath)
+	if err != nil {
+		return nil, "", fmt.Errorf("error while loading file: %w", err)
+	}
+
+	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
+	}
+	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	if !ok {
+		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+
+	return config, contentHash, validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
+}
+
+func loadDataAndHash(filepath string) ([]byte, string, error) {
 	f, err := os.Open(filepath)
 	if err != nil {
 		return nil, "", fmt.Errorf("error opening encryption provider configuration file %q: %w", filepath, err)
@@ -482,27 +536,20 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
 		return nil, "", fmt.Errorf("encryption provider configuration file %q is empty", filepath)
 	}
 
-	scheme := runtime.NewScheme()
-	codecs := serializer.NewCodecFactory(scheme)
-	utilruntime.Must(apiserverconfig.AddToScheme(scheme))
-	utilruntime.Must(apiserverconfigv1.AddToScheme(scheme))
+	return data, computeEncryptionConfigHash(data), nil
+}
 
-	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
-	if err != nil {
-		return nil, "", fmt.Errorf("error decoding encryption provider configuration file %q: %w", filepath, err)
-	}
-	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
-	if !ok {
-		return nil, "", fmt.Errorf("got unexpected config type: %v", gvk)
-	}
-
-	return config, computeEncryptionConfigHash(data), validation.ValidateEncryptionConfiguration(config, reload).ToAggregate()
+// GetEncryptionConfigHash reads the encryption configuration file at filepath and returns the hash of the file.
+// It does not attempt to decode or load the config, and serves as a cheap check to determine if the file has changed.
+func GetEncryptionConfigHash(filepath string) (string, error) {
+	_, contentHash, err := loadDataAndHash(filepath)
+	return contentHash, err
 }
 
 // prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
+func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration, apiServerID string) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
 	var transformers []storagevalue.PrefixTransformer
 	var probes []healthChecker
 	var kmsUsed kmsState
@@ -530,7 +577,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
 			transformer, transformerErr = secretboxPrefixTransformer(provider.Secretbox)
 
 		case provider.KMS != nil:
-			transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS)
+			transformer, probe, used, transformerErr = kmsPrefixTransformer(ctx, provider.KMS, apiServerID)
 			if transformerErr == nil {
 				probes = append(probes, probe)
 				kmsUsed.accumulate(used)
@@ -689,7 +736,7 @@ func (s *kmsState) accumulate(other *kmsState) {
 // kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
 // It may launch multiple go routines whose lifecycle is controlled by ctx.
 // In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
-func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
+func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration, apiServerID string) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
 	kmsName := config.Name
 	switch config.APIVersion {
 	case kmsAPIVersionV1:
@@ -735,14 +782,14 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
 			service:      envelopeService,
 			l:            &sync.Mutex{},
 			lastResponse: &kmsPluginHealthzResponse{},
+			apiServerID:  apiServerID,
 		}
 		// initialize state so that Load always works
 		probe.state.Store(&envelopekmsv2.State{})
 
 		primeAndProbeKMSv2(ctx, probe, kmsName)
-
 		transformer := storagevalue.PrefixTransformer{
-			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState),
+			Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState, apiServerID),
 			Prefix:      []byte(kmsTransformerPrefixV2 + kmsName + ":"),
 		}
 

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go

@@ -20,9 +20,9 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"sync"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
@@ -35,8 +35,11 @@ import (
 // workqueueKey is the dummy key used to process change in encryption config file.
 const workqueueKey = "key"
 
-// DynamicKMSEncryptionConfigContent which can dynamically handle changes in encryption config file.
-type DynamicKMSEncryptionConfigContent struct {
+// EncryptionConfigFileChangePollDuration is exposed so that integration tests can crank up the reload speed.
+var EncryptionConfigFileChangePollDuration = time.Minute
+
+// DynamicEncryptionConfigContent which can dynamically handle changes in encryption config file.
+type DynamicEncryptionConfigContent struct {
 	name string
 
 	// filePath is the path of the file to read.
@@ -50,6 +53,17 @@ type DynamicKMSEncryptionConfigContent struct {
 
 	// dynamicTransformers updates the transformers when encryption config file changes.
 	dynamicTransformers *encryptionconfig.DynamicTransformers
+
+	// identity of the api server
+	apiServerID string
+
+	// can be swapped during testing
+	getEncryptionConfigHash func(ctx context.Context, filepath string) (string, error)
+	loadEncryptionConfig    func(ctx context.Context, filepath string, reload bool, apiServerID string) (*encryptionconfig.EncryptionConfiguration, error)
+}
+
+func init() {
+	metrics.RegisterMetrics()
 }
 
 // NewDynamicEncryptionConfiguration returns controller that dynamically reacts to changes in encryption config file.
@@ -57,94 +71,73 @@ func NewDynamicEncryptionConfiguration(
 	name, filePath string,
 	dynamicTransformers *encryptionconfig.DynamicTransformers,
 	configContentHash string,
-) *DynamicKMSEncryptionConfigContent {
-	encryptionConfig := &DynamicKMSEncryptionConfigContent{
+	apiServerID string,
+) *DynamicEncryptionConfigContent {
+	return &DynamicEncryptionConfigContent{
 		name:                           name,
 		filePath:                       filePath,
 		lastLoadedEncryptionConfigHash: configContentHash,
 		dynamicTransformers:            dynamicTransformers,
 		queue:                          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), name),
+		apiServerID:                    apiServerID,
+		getEncryptionConfigHash: func(_ context.Context, filepath string) (string, error) {
+			return encryptionconfig.GetEncryptionConfigHash(filepath)
+		},
+		loadEncryptionConfig: encryptionconfig.LoadEncryptionConfig,
 	}
-	encryptionConfig.queue.Add(workqueueKey) // to avoid missing any file changes that occur in between the initial load and Run
-
-	return encryptionConfig
 }
 
-// Run starts the controller and blocks until stopCh is closed.
-func (d *DynamicKMSEncryptionConfigContent) Run(ctx context.Context) {
+// Run starts the controller and blocks until ctx is canceled.
+func (d *DynamicEncryptionConfigContent) Run(ctx context.Context) {
 	defer utilruntime.HandleCrash()
-	defer d.queue.ShutDown()
 
 	klog.InfoS("Starting controller", "name", d.name)
 	defer klog.InfoS("Shutting down controller", "name", d.name)
 
-	// start worker for processing content
-	go wait.UntilWithContext(ctx, d.runWorker, time.Second)
+	var wg sync.WaitGroup
 
-	// start the loop that watches the encryption config file until stopCh is closed.
-	go wait.UntilWithContext(ctx, func(ctx context.Context) {
-		if err := d.watchEncryptionConfigFile(ctx); err != nil {
-			// if there is an error while setting up or handling the watches, this will ensure that we will process the config file.
-			defer d.queue.Add(workqueueKey)
-			klog.ErrorS(err, "Failed to watch encryption config file, will retry later")
-		}
-	}, time.Second)
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		defer d.queue.ShutDown()
+		<-ctx.Done()
+	}()
 
-	<-ctx.Done()
-}
+	wg.Add(1)
+	go func() {
+		defer utilruntime.HandleCrash()
+		defer wg.Done()
+		d.runWorker(ctx)
+	}()
 
-func (d *DynamicKMSEncryptionConfigContent) watchEncryptionConfigFile(ctx context.Context) error {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return fmt.Errorf("error creating fsnotify watcher: %w", err)
-	}
-	defer watcher.Close()
+	// this function polls changes in the encryption config file by placing a dummy key in the queue.
+	// the 'runWorker' function then picks up this dummy key and processes the changes.
+	// the goroutine terminates when 'ctx' is canceled.
+	_ = wait.PollUntilContextCancel(
+		ctx,
+		EncryptionConfigFileChangePollDuration,
+		true,
+		func(ctx context.Context) (bool, error) {
+			// add dummy item to the queue to trigger file content processing.
+			d.queue.Add(workqueueKey)
 
-	if err = watcher.Add(d.filePath); err != nil {
-		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
-	}
+			// return false to continue polling.
+			return false, nil
+		},
+	)
 
-	for {
-		select {
-		case event := <-watcher.Events:
-			if err := d.handleWatchEvent(event, watcher); err != nil {
-				return err
-			}
-		case err := <-watcher.Errors:
-			return fmt.Errorf("received fsnotify error: %w", err)
-		case <-ctx.Done():
-			return nil
-		}
-	}
-}
-
-func (d *DynamicKMSEncryptionConfigContent) handleWatchEvent(event fsnotify.Event, watcher *fsnotify.Watcher) error {
-	// This should be executed after restarting the watch (if applicable) to ensure no file event will be missing.
-	defer d.queue.Add(workqueueKey)
-
-	// return if file has not been removed or renamed.
-	if event.Op&(fsnotify.Remove|fsnotify.Rename) == 0 {
-		return nil
-	}
-
-	if err := watcher.Remove(d.filePath); err != nil {
-		klog.V(2).InfoS("Failed to remove file watch, it may have been deleted", "file", d.filePath, "err", err)
-	}
-	if err := watcher.Add(d.filePath); err != nil {
-		return fmt.Errorf("error adding watch for file %s: %w", d.filePath, err)
-	}
-
-	return nil
+	wg.Wait()
 }
 
 // runWorker to process file content
-func (d *DynamicKMSEncryptionConfigContent) runWorker(ctx context.Context) {
+func (d *DynamicEncryptionConfigContent) runWorker(ctx context.Context) {
 	for d.processNextWorkItem(ctx) {
 	}
 }
 
 // processNextWorkItem processes file content when there is a message in the queue.
-func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
+func (d *DynamicEncryptionConfigContent) processNextWorkItem(serverCtx context.Context) bool {
 	// key here is dummy item in the queue to trigger file content processing.
 	key, quit := d.queue.Get()
 	if quit {
@@ -152,6 +145,12 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	}
 	defer d.queue.Done(key)
 
+	d.processWorkItem(serverCtx, key)
+
+	return true
+}
+
+func (d *DynamicEncryptionConfigContent) processWorkItem(serverCtx context.Context, workqueueKey interface{}) {
 	var (
 		updatedEffectiveConfig  bool
 		err                     error
@@ -172,32 +171,32 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 		}
 
 		if updatedEffectiveConfig && err == nil {
-			metrics.RecordEncryptionConfigAutomaticReloadSuccess()
+			metrics.RecordEncryptionConfigAutomaticReloadSuccess(d.apiServerID)
 		}
 
 		if err != nil {
-			metrics.RecordEncryptionConfigAutomaticReloadFailure()
+			metrics.RecordEncryptionConfigAutomaticReloadFailure(d.apiServerID)
 			utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
 			// add dummy item back to the queue to trigger file content processing.
-			d.queue.AddRateLimited(key)
+			d.queue.AddRateLimited(workqueueKey)
 		}
 	}()
 
 	encryptionConfiguration, configChanged, err = d.processEncryptionConfig(ctx)
 	if err != nil {
-		return true
+		return
 	}
 	if !configChanged {
-		return true
+		return
 	}
 
 	if len(encryptionConfiguration.HealthChecks) != 1 {
 		err = fmt.Errorf("unexpected number of healthz checks: %d. Should have only one", len(encryptionConfiguration.HealthChecks))
-		return true
+		return
 	}
 	// get healthz checks for all new KMS plugins.
 	if err = d.validateNewTransformersHealth(ctx, encryptionConfiguration.HealthChecks[0], encryptionConfiguration.KMSCloseGracePeriod); err != nil {
-		return true
+		return
 	}
 
 	// update transformers.
@@ -214,30 +213,44 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
 	klog.V(2).InfoS("Loaded new kms encryption config content", "name", d.name)
 
 	updatedEffectiveConfig = true
-	return true
 }
 
 // loadEncryptionConfig processes the next set of content from the file.
-func (d *DynamicKMSEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
-	encryptionConfiguration *encryptionconfig.EncryptionConfiguration,
+func (d *DynamicEncryptionConfigContent) processEncryptionConfig(ctx context.Context) (
+	_ *encryptionconfig.EncryptionConfiguration,
 	configChanged bool,
-	err error,
+	_ error,
 ) {
-	// this code path will only execute if reload=true. So passing true explicitly.
-	encryptionConfiguration, err = encryptionconfig.LoadEncryptionConfig(ctx, d.filePath, true)
+	contentHash, err := d.getEncryptionConfigHash(ctx, d.filePath)
 	if err != nil {
 		return nil, false, err
 	}
 
 	// check if encryptionConfig is different from the current. Do nothing if they are the same.
-	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
-		klog.V(4).InfoS("Encryption config has not changed", "name", d.name)
+	if contentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (before load)", "name", d.name)
 		return nil, false, nil
 	}
+
+	// this code path will only execute if reload=true. So passing true explicitly.
+	encryptionConfiguration, err := d.loadEncryptionConfig(ctx, d.filePath, true, d.apiServerID)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// check if encryptionConfig is different from the current (again to avoid TOCTOU). Do nothing if they are the same.
+	if encryptionConfiguration.EncryptionFileContentHash == d.lastLoadedEncryptionConfigHash {
+		klog.V(4).InfoS("Encryption config has not changed (after load)", "name", d.name)
+		return nil, false, nil
+	}
+
 	return encryptionConfiguration, true, nil
 }
 
-func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
+// minKMSPluginCloseGracePeriod can be lowered in unit tests to make the health check poll faster
+var minKMSPluginCloseGracePeriod = 10 * time.Second
+
+func (d *DynamicEncryptionConfigContent) validateNewTransformersHealth(
 	ctx context.Context,
 	kmsPluginHealthzCheck healthz.HealthChecker,
 	kmsPluginCloseGracePeriod time.Duration,
@@ -245,8 +258,8 @@ func (d *DynamicKMSEncryptionConfigContent) validateNewTransformersHealth(
 	// test if new transformers are healthy
 	var healthCheckError error
 
-	if kmsPluginCloseGracePeriod < 10*time.Second {
-		kmsPluginCloseGracePeriod = 10 * time.Second
+	if kmsPluginCloseGracePeriod < minKMSPluginCloseGracePeriod {
+		kmsPluginCloseGracePeriod = minKMSPluginCloseGracePeriod
 	}
 
 	// really make sure that the immediate check does not hang

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go

@@ -17,6 +17,9 @@ limitations under the License.
 package metrics
 
 import (
+	"crypto/sha256"
+	"fmt"
+	"hash"
 	"sync"
 
 	"k8s.io/component-base/metrics"
@@ -29,24 +32,26 @@ const (
 )
 
 var (
-	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
+	encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_failures_total",
-			Help:           "Total number of failed automatic reloads of encryption configuration.",
+			Help:           "Total number of failed automatic reloads of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
+		[]string{"apiserver_id_hash"},
 	)
 
-	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
+	encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounterVec(
 		&metrics.CounterOpts{
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_success_total",
-			Help:           "Total number of successful automatic reloads of encryption configuration.",
+			Help:           "Total number of successful automatic reloads of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
+		[]string{"apiserver_id_hash"},
 	)
 
 	encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
@@ -54,33 +59,53 @@ var (
 			Namespace:      namespace,
 			Subsystem:      subsystem,
 			Name:           "automatic_reload_last_timestamp_seconds",
-			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration.",
+			Help:           "Timestamp of the last successful or failed automatic reload of encryption configuration split by apiserver identity.",
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"status"},
+		[]string{"status", "apiserver_id_hash"},
 	)
 )
 
 var registerMetrics sync.Once
+var hashPool *sync.Pool
 
 func RegisterMetrics() {
 	registerMetrics.Do(func() {
+		hashPool = &sync.Pool{
+			New: func() interface{} {
+				return sha256.New()
+			},
+		}
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
 		legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
 	})
 }
 
-func RecordEncryptionConfigAutomaticReloadFailure() {
-	encryptionConfigAutomaticReloadFailureTotal.Inc()
-	recordEncryptionConfigAutomaticReloadTimestamp("failure")
+func RecordEncryptionConfigAutomaticReloadFailure(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadFailureTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("failure", apiServerIDHash)
 }
 
-func RecordEncryptionConfigAutomaticReloadSuccess() {
-	encryptionConfigAutomaticReloadSuccessTotal.Inc()
-	recordEncryptionConfigAutomaticReloadTimestamp("success")
+func RecordEncryptionConfigAutomaticReloadSuccess(apiServerID string) {
+	apiServerIDHash := getHash(apiServerID)
+	encryptionConfigAutomaticReloadSuccessTotal.WithLabelValues(apiServerIDHash).Inc()
+	recordEncryptionConfigAutomaticReloadTimestamp("success", apiServerIDHash)
 }
 
-func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
-	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
+func recordEncryptionConfigAutomaticReloadTimestamp(result, apiServerIDHash string) {
+	encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result, apiServerIDHash).SetToCurrentTime()
+}
+
+func getHash(data string) string {
+	if len(data) == 0 {
+		return ""
+	}
+	h := hashPool.Get().(hash.Hash)
+	h.Reset()
+	h.Write([]byte(data))
+	dataHash := fmt.Sprintf("sha256:%x", h.Sum(nil))
+	hashPool.Put(h)
+	return dataHash
 }

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/spf13/pflag"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -44,8 +45,6 @@ import (
 )
 
 type EtcdOptions struct {
-	// The value of Paging on StorageConfig will be overridden by the
-	// calculated feature gate value.
 	StorageConfig                           storagebackend.Config
 	EncryptionProviderConfigFilepath        string
 	EncryptionProviderConfigAutomaticReload bool
@@ -87,6 +86,12 @@ func NewEtcdOptions(backendConfig *storagebackend.Config) *EtcdOptions {
 	return options
 }
 
+var storageMediaTypes = sets.New(
+	runtime.ContentTypeJSON,
+	runtime.ContentTypeYAML,
+	runtime.ContentTypeProtobuf,
+)
+
 func (s *EtcdOptions) Validate() []error {
 	if s == nil {
 		return nil
@@ -120,6 +125,10 @@ func (s *EtcdOptions) Validate() []error {
 		allErrors = append(allErrors, fmt.Errorf("--encryption-provider-config-automatic-reload must be set with --encryption-provider-config"))
 	}
 
+	if s.DefaultStorageMediaType != "" && !storageMediaTypes.Has(s.DefaultStorageMediaType) {
+		allErrors = append(allErrors, fmt.Errorf("--storage-media-type %q invalid, allowed values: %s", s.DefaultStorageMediaType, strings.Join(sets.List(storageMediaTypes), ", ")))
+	}
+
 	return allErrors
 }
 
@@ -294,7 +303,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 		}
 	}()
 
-	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload)
+	encryptionConfiguration, err := encryptionconfig.LoadEncryptionConfig(ctxTransformers, s.EncryptionProviderConfigFilepath, s.EncryptionProviderConfigAutomaticReload, c.APIServerID)
 	if err != nil {
 		return err
 	}
@@ -318,6 +327,7 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 					s.EncryptionProviderConfigFilepath,
 					dynamicTransformers,
 					encryptionConfiguration.EncryptionFileContentHash,
+					c.APIServerID,
 				)
 
 				go dynamicEncryptionConfigController.Run(ctxServer)
@@ -331,18 +341,23 @@ func (s *EtcdOptions) maybeApplyResourceTransformers(c *server.Config) (err erro
 
 		c.ResourceTransformers = dynamicTransformers
 		if !s.SkipHealthEndpoints {
-			c.AddHealthChecks(dynamicTransformers)
+			addHealthChecksWithoutLivez(c, dynamicTransformers)
 		}
 	} else {
 		c.ResourceTransformers = encryptionconfig.StaticTransformers(encryptionConfiguration.Transformers)
 		if !s.SkipHealthEndpoints {
-			c.AddHealthChecks(encryptionConfiguration.HealthChecks...)
+			addHealthChecksWithoutLivez(c, encryptionConfiguration.HealthChecks...)
 		}
 	}
 
 	return nil
 }
 
+func addHealthChecksWithoutLivez(c *server.Config, healthChecks ...healthz.HealthChecker) {
+	c.HealthzChecks = append(c.HealthzChecks, healthChecks...)
+	c.ReadyzChecks = append(c.ReadyzChecks, healthChecks...)
+}
+
 func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 	healthCheck, err := storagefactory.CreateHealthCheck(s.StorageConfig, c.DrainedNotify())
 	if err != nil {

vendor/k8s.io/apiserver/pkg/server/options/feature.go

@@ -17,16 +17,22 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
+
 	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apiserver/pkg/server"
+	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
 )
 
 type FeatureOptions struct {
 	EnableProfiling           bool
 	DebugSocketPath           string
 	EnableContentionProfiling bool
+	EnablePriorityAndFairness bool
 }
 
 func NewFeatureOptions() *FeatureOptions {
@@ -36,6 +42,7 @@ func NewFeatureOptions() *FeatureOptions {
 		EnableProfiling:           defaults.EnableProfiling,
 		DebugSocketPath:           defaults.DebugSocketPath,
 		EnableContentionProfiling: defaults.EnableContentionProfiling,
+		EnablePriorityAndFairness: true,
 	}
 }
 
@@ -50,9 +57,11 @@ func (o *FeatureOptions) AddFlags(fs *pflag.FlagSet) {
 		"Enable block profiling, if profiling is enabled")
 	fs.StringVar(&o.DebugSocketPath, "debug-socket-path", o.DebugSocketPath,
 		"Use an unprotected (no authn/authz) unix-domain socket for profiling with the given path")
+	fs.BoolVar(&o.EnablePriorityAndFairness, "enable-priority-and-fairness", o.EnablePriorityAndFairness, ""+
+		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
 }
 
-func (o *FeatureOptions) ApplyTo(c *server.Config) error {
+func (o *FeatureOptions) ApplyTo(c *server.Config, clientset kubernetes.Interface, informers informers.SharedInformerFactory) error {
 	if o == nil {
 		return nil
 	}
@@ -61,6 +70,18 @@ func (o *FeatureOptions) ApplyTo(c *server.Config) error {
 	c.DebugSocketPath = o.DebugSocketPath
 	c.EnableContentionProfiling = o.EnableContentionProfiling
 
+	if o.EnablePriorityAndFairness {
+		if c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight <= 0 {
+			return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight)
+
+		}
+		c.FlowControl = utilflowcontrol.New(
+			informers,
+			clientset.FlowcontrolV1(),
+			c.MaxRequestsInFlight+c.MaxMutatingRequestsInFlight,
+		)
+	}
+
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -17,20 +17,15 @@ limitations under the License.
 package options
 
 import (
-	"fmt"
-
 	"github.com/spf13/pflag"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/util/feature"
-	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/featuregate"
-	"k8s.io/klog/v2"
 )
 
 // RecommendedOptions contains the recommended options for running an API server.
@@ -122,17 +117,17 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Audit.ApplyTo(&config.Config); err != nil {
 		return err
 	}
-	if err := o.Features.ApplyTo(&config.Config); err != nil {
-		return err
-	}
 	if err := o.CoreAPI.ApplyTo(config); err != nil {
 		return err
 	}
-	initializers, err := o.ExtraAdmissionInitializers(config)
+	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
 	if err != nil {
 		return err
 	}
-	kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
+	if err := o.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
+		return err
+	}
+	initializers, err := o.ExtraAdmissionInitializers(config)
 	if err != nil {
 		return err
 	}
@@ -144,22 +139,6 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 		initializers...); err != nil {
 		return err
 	}
-	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
-		if config.ClientConfig != nil {
-			if config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight <= 0 {
-				return fmt.Errorf("invalid configuration: MaxRequestsInFlight=%d and MaxMutatingRequestsInFlight=%d; they must add up to something positive", config.MaxRequestsInFlight, config.MaxMutatingRequestsInFlight)
-
-			}
-			config.FlowControl = utilflowcontrol.New(
-				config.SharedInformerFactory,
-				kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta3(),
-				config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
-				config.RequestTimeout/4,
-			)
-		} else {
-			klog.Warningf("Neither kubeconfig is provided nor service-account is mounted, so APIPriorityAndFairness will be disabled")
-		}
-	}
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -62,8 +62,7 @@ type ServerRunOptions struct {
 	// decoded in a write request. 0 means no limit.
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
-	MaxRequestBodyBytes       int64
-	EnablePriorityAndFairness bool
+	MaxRequestBodyBytes int64
 
 	// ShutdownSendRetryAfter dictates when to initiate shutdown of the HTTP
 	// Server during the graceful termination of the apiserver. If true, we wait
@@ -104,7 +103,6 @@ func NewServerRunOptions() *ServerRunOptions {
 		ShutdownWatchTerminationGracePeriod: defaults.ShutdownWatchTerminationGracePeriod,
 		JSONPatchMaxCopyBytes:               defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:                 defaults.MaxRequestBodyBytes,
-		EnablePriorityAndFairness:           true,
 		ShutdownSendRetryAfter:              false,
 	}
 }
@@ -325,9 +323,6 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
-	fs.BoolVar(&s.EnablePriorityAndFairness, "enable-priority-and-fairness", s.EnablePriorityAndFairness, ""+
-		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
-
 	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
 		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
 		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -260,7 +260,39 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	c := *config
 
 	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
-	// load main cert
+	// load main cert *original description until 2023-08-18*
+
+	/*
+		kubernetes mutual (2-way) x509 between client and apiserver:
+
+			>1. apiserver sending its apiserver certificate along with its publickey to client
+			2. client verifies the apiserver certificate sent against its cluster certificate authority data
+			3. client sending its client certificate along with its public key to the apiserver
+			4. apiserver verifies the client certificate sent against its cluster certificate authority data
+
+			description:
+				here, with this block,
+				apiserver certificate and pub key data (along with priv key)get loaded into server.SecureServingInfo
+				for client to later in the step 2 verify the apiserver certificate during the handshake
+				when making a request
+
+			normal args related to this stage:
+				--tls-cert-file string  File containing the default x509 Certificate for HTTPS.
+					(CA cert, if any, concatenated after server cert). If HTTPS serving is enabled, and
+					--tls-cert-file and --tls-private-key-file are not provided, a self-signed certificate
+					and key are generated for the public address and saved to the directory specified by
+					--cert-dir
+				--tls-private-key-file string  File containing the default x509 private key matching --tls-cert-file.
+
+				(retrievable from "kube-apiserver --help" command)
+				(suggested by @deads2k)
+
+			see also:
+				- for the step 2, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 3, see: staging/src/k8s.io/client-go/transport/transport.go
+				- for the step 4, see: staging/src/k8s.io/apiserver/pkg/authentication/request/x509/x509.go
+	*/
+
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
 		var err error
 		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)