rebase: update all k8s packages to 0.27.2

Signed-off-by: Niels de Vos <ndevos@ibm.com>

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go

@@ -23,14 +23,24 @@ import (
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/rand"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
+	"sync/atomic"
+	"time"
 
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/klog/v2"
 )
 
-// gcm implements AEAD encryption of the provided values given a cipher.Block algorithm.
+type gcm struct {
+	aead      cipher.AEAD
+	nonceFunc func([]byte) error
+}
+
+// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given data.
+// It implements AEAD encryption of the provided values given a cipher.Block algorithm.
 // The authenticated data provided as part of the value.Context method must match when the same
 // value is set to and loaded from storage. In order to ensure that values cannot be copied by
 // an attacker from a location under their control, use characteristics of the storage location
@@ -43,44 +53,148 @@ import (
 // therefore transformers using this implementation *must* ensure they allow for frequent key
 // rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
 // random nonces.
-type gcm struct {
-	block cipher.Block
+func NewGCMTransformer(block cipher.Block) (value.Transformer, error) {
+	aead, err := newGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	return &gcm{aead: aead, nonceFunc: randomNonce}, nil
 }
 
-// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given
-// data.
-func NewGCMTransformer(block cipher.Block) value.Transformer {
-	return &gcm{block: block}
+// NewGCMTransformerWithUniqueKeyUnsafe is the same as NewGCMTransformer but is unsafe for general
+// use because it makes assumptions about the key underlying the block cipher.  Specifically,
+// it uses a 96-bit nonce where the first 32 bits are random data and the remaining 64 bits are
+// a monotonically incrementing atomic counter.  This means that the key must be randomly generated
+// on process startup and must never be used for encryption outside the lifetime of the process.
+// Unlike NewGCMTransformer, this function is immune to the birthday attack and thus the key can
+// be used for 2^64-1 writes without rotation.  Furthermore, cryptographic wear out of AES-GCM with
+// a sequential nonce occurs after 2^64 encryptions, which is not a concern for our use cases.
+// Even if that occurs, the nonce counter would overflow and crash the process.  We have no concerns
+// around plaintext length because all stored items are small (less than 2 MB).  To prevent the
+// chance of the block cipher being accidentally re-used, it is not taken in as input.  Instead,
+// a new random key is generated and returned on every invocation of this function.  This key is
+// used as the input to the block cipher.  If the key is stored and retrieved at a later point,
+// it can be passed to NewGCMTransformer(aes.NewCipher(key)) to construct a transformer capable
+// of decrypting values encrypted by this transformer (that transformer must not be used for encryption).
+func NewGCMTransformerWithUniqueKeyUnsafe() (value.Transformer, []byte, error) {
+	key, err := generateKey(32)
+	if err != nil {
+		return nil, nil, err
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	nonceGen := &nonceGenerator{
+		// we start the nonce counter at one billion so that we are
+		// guaranteed to detect rollover across different go routines
+		zero:  1_000_000_000,
+		fatal: die,
+	}
+	nonceGen.nonce.Add(nonceGen.zero)
+
+	transformer, err := newGCMTransformerWithUniqueKeyUnsafe(block, nonceGen)
+	if err != nil {
+		return nil, nil, err
+	}
+	return transformer, key, nil
+}
+
+func newGCMTransformerWithUniqueKeyUnsafe(block cipher.Block, nonceGen *nonceGenerator) (value.Transformer, error) {
+	aead, err := newGCM(block)
+	if err != nil {
+		return nil, err
+	}
+
+	nonceFunc := func(b []byte) error {
+		// we only need 8 bytes to store our 64 bit incrementing nonce
+		// instead of leaving the unused bytes as zeros, set those to random bits
+		// this mostly protects us from weird edge cases like a VM restore that rewinds our atomic counter
+		randNonceSize := len(b) - 8
+
+		if err := randomNonce(b[:randNonceSize]); err != nil {
+			return err
+		}
+
+		nonceGen.next(b[randNonceSize:])
+
+		return nil
+	}
+
+	return &gcm{aead: aead, nonceFunc: nonceFunc}, nil
+}
+
+func newGCM(block cipher.Block) (cipher.AEAD, error) {
+	aead, err := cipher.NewGCM(block)
+	if err != nil {
+		return nil, err
+	}
+	if nonceSize := aead.NonceSize(); nonceSize != 12 { // all data in etcd will be broken if this ever changes
+		return nil, fmt.Errorf("crypto/cipher.NewGCM returned unexpected nonce size: %d", nonceSize)
+	}
+	return aead, nil
+}
+
+func randomNonce(b []byte) error {
+	_, err := rand.Read(b)
+	return err
+}
+
+type nonceGenerator struct {
+	// even at one million encryptions per second, this counter is enough for half a million years
+	// using this struct avoids alignment bugs: https://pkg.go.dev/sync/atomic#pkg-note-BUG
+	nonce atomic.Uint64
+	zero  uint64
+	fatal func(msg string)
+}
+
+func (n *nonceGenerator) next(b []byte) {
+	incrementingNonce := n.nonce.Add(1)
+	if incrementingNonce <= n.zero {
+		// this should never happen, and is unrecoverable if it does
+		n.fatal("aes-gcm detected nonce overflow - cryptographic wear out has occurred")
+	}
+	binary.LittleEndian.PutUint64(b, incrementingNonce)
+}
+
+func die(msg string) {
+	// nolint:logcheck // we want the stack traces, log flushing, and process exiting logic from FatalDepth
+	klog.FatalDepth(1, msg)
+}
+
+// generateKey generates a random key using system randomness.
+func generateKey(length int) (key []byte, err error) {
+	defer func(start time.Time) {
+		value.RecordDataKeyGeneration(start, err)
+	}(time.Now())
+	key = make([]byte, length)
+	if _, err = rand.Read(key); err != nil {
+		return nil, err
+	}
+
+	return key, nil
 }
 
 func (t *gcm) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
-	aead, err := cipher.NewGCM(t.block)
-	if err != nil {
-		return nil, false, err
-	}
-	nonceSize := aead.NonceSize()
+	nonceSize := t.aead.NonceSize()
 	if len(data) < nonceSize {
-		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+		return nil, false, errors.New("the stored data was shorter than the required size")
 	}
-	result, err := aead.Open(nil, data[:nonceSize], data[nonceSize:], dataCtx.AuthenticatedData())
+	result, err := t.aead.Open(nil, data[:nonceSize], data[nonceSize:], dataCtx.AuthenticatedData())
 	return result, false, err
 }
 
 func (t *gcm) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
-	aead, err := cipher.NewGCM(t.block)
-	if err != nil {
-		return nil, err
+	nonceSize := t.aead.NonceSize()
+	result := make([]byte, nonceSize+t.aead.Overhead()+len(data))
+
+	if err := t.nonceFunc(result[:nonceSize]); err != nil {
+		return nil, fmt.Errorf("failed to write nonce for AES-GCM: %w", err)
 	}
-	nonceSize := aead.NonceSize()
-	result := make([]byte, nonceSize+aead.Overhead()+len(data))
-	n, err := rand.Read(result[:nonceSize])
-	if err != nil {
-		return nil, err
-	}
-	if n != nonceSize {
-		return nil, fmt.Errorf("unable to read sufficient random bytes")
-	}
-	cipherText := aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, dataCtx.AuthenticatedData())
+
+	cipherText := t.aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, dataCtx.AuthenticatedData())
 	return result[:nonceSize+len(cipherText)], nil
 }
 
@@ -96,7 +210,7 @@ func NewCBCTransformer(block cipher.Block) value.Transformer {
 }
 
 var (
-	ErrInvalidBlockSize    = fmt.Errorf("the stored data is not a multiple of the block size")
+	errInvalidBlockSize    = errors.New("the stored data is not a multiple of the block size")
 	errInvalidPKCS7Data    = errors.New("invalid PKCS7 data (empty or not padded)")
 	errInvalidPKCS7Padding = errors.New("invalid padding on input")
 )
@@ -104,13 +218,13 @@ var (
 func (t *cbc) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
 	blockSize := aes.BlockSize
 	if len(data) < blockSize {
-		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+		return nil, false, errors.New("the stored data was shorter than the required size")
 	}
 	iv := data[:blockSize]
 	data = data[blockSize:]
 
 	if len(data)%blockSize != 0 {
-		return nil, false, ErrInvalidBlockSize
+		return nil, false, errInvalidBlockSize
 	}
 
 	result := make([]byte, len(data))
@@ -140,7 +254,7 @@ func (t *cbc) TransformToStorage(ctx context.Context, data []byte, dataCtx value
 	result := make([]byte, blockSize+len(data)+paddingSize)
 	iv := result[:blockSize]
 	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
-		return nil, fmt.Errorf("unable to read sufficient random bytes")
+		return nil, errors.New("unable to read sufficient random bytes")
 	}
 	copy(result[blockSize:], data)
 

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go

@@ -53,7 +53,7 @@ type envelopeTransformer struct {
 	transformers *lru.Cache
 
 	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
-	baseTransformerFunc func(cipher.Block) value.Transformer
+	baseTransformerFunc func(cipher.Block) (value.Transformer, error)
 
 	cacheSize    int
 	cacheEnabled bool
@@ -63,7 +63,7 @@ type envelopeTransformer struct {
 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
 // the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
 // used decrypted DEKs in memory.
-func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer {
+func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) (value.Transformer, error)) value.Transformer {
 	var (
 		cache *lru.Cache
 	)
@@ -161,7 +161,11 @@ func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.T
 	if err != nil {
 		return nil, err
 	}
-	transformer := t.baseTransformerFunc(block)
+	transformer, err := t.baseTransformerFunc(block)
+	if err != nil {
+		return nil, err
+	}
+
 	// Use base64 of encKey as the key into the cache because hashicorp/golang-lru
 	// cannot hash []uint8.
 	if t.cacheEnabled {

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go

@@ -28,9 +28,9 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/util"
 	"k8s.io/klog/v2"
 	kmsapi "k8s.io/kms/apis/v1beta1"
+	"k8s.io/kms/pkg/util"
 )
 
 const (
@@ -53,7 +53,7 @@ type gRPCService struct {
 
 // NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider.
 func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Duration) (Service, error) {
-	klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint)
+	klog.V(4).InfoS("Configure KMS provider", "endpoint", endpoint)
 
 	addr, err := util.ParseEndpoint(endpoint)
 	if err != nil {
@@ -72,9 +72,9 @@ func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Durat
 				// addr - comes from the closure
 				c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr})
 				if err != nil {
-					klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
+					klog.ErrorS(err, "failed to create connection to unix socket", "addr", addr)
 				} else {
-					klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
+					klog.V(4).InfoS("Successfully dialed Unix socket", "addr", addr)
 				}
 				return c, err
 			}))
@@ -113,7 +113,7 @@ func (g *gRPCService) checkAPIVersion(ctx context.Context) error {
 	}
 	g.versionChecked = true
 
-	klog.V(4).Infof("Version of KMS provider is %s", response.Version)
+	klog.V(4).InfoS("KMS provider api version verified", "version", response.Version)
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/cache.go

@@ -0,0 +1,108 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kmsv2 transforms values for storage at rest using a Envelope v2 provider
+package kmsv2
+
+import (
+	"context"
+	"crypto/sha256"
+	"hash"
+	"sync"
+	"time"
+	"unsafe"
+
+	utilcache "k8s.io/apimachinery/pkg/util/cache"
+	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/utils/clock"
+)
+
+// prevent decryptTransformer from drifting from value.Transformer
+var _ decryptTransformer = value.Transformer(nil)
+
+// decryptTransformer is the decryption subset of value.Transformer.
+// this exists purely to statically enforce that transformers placed in the cache are not used for encryption.
+// this is relevant in the context of nonce collision since transformers that are created
+// from encrypted DEKs retrieved from etcd cannot maintain their nonce counter state.
+type decryptTransformer interface {
+	TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error)
+}
+
+type simpleCache struct {
+	cache *utilcache.Expiring
+	ttl   time.Duration
+	// hashPool is a per cache pool of hash.Hash (to avoid allocations from building the Hash)
+	// SHA-256 is used to prevent collisions
+	hashPool *sync.Pool
+}
+
+func newSimpleCache(clock clock.Clock, ttl time.Duration) *simpleCache {
+	return &simpleCache{
+		cache: utilcache.NewExpiringWithClock(clock),
+		ttl:   ttl,
+		hashPool: &sync.Pool{
+			New: func() interface{} {
+				return sha256.New()
+			},
+		},
+	}
+}
+
+// given a key, return the transformer, or nil if it does not exist in the cache
+func (c *simpleCache) get(key []byte) decryptTransformer {
+	record, ok := c.cache.Get(c.keyFunc(key))
+	if !ok {
+		return nil
+	}
+	return record.(decryptTransformer)
+}
+
+// set caches the record for the key
+func (c *simpleCache) set(key []byte, transformer decryptTransformer) {
+	if len(key) == 0 {
+		panic("key must not be empty")
+	}
+	if transformer == nil {
+		panic("transformer must not be nil")
+	}
+	c.cache.Set(c.keyFunc(key), transformer, c.ttl)
+}
+
+// keyFunc generates a string key by hashing the inputs.
+// This lowers the memory requirement of the cache.
+func (c *simpleCache) keyFunc(s []byte) string {
+	h := c.hashPool.Get().(hash.Hash)
+	h.Reset()
+
+	if _, err := h.Write(s); err != nil {
+		panic(err) // Write() on hash never fails
+	}
+	key := toString(h.Sum(nil)) // skip base64 encoding to save an allocation
+	c.hashPool.Put(h)
+
+	return key
+}
+
+// toString performs unholy acts to avoid allocations
+func toString(b []byte) string {
+	// unsafe.SliceData relies on cap whereas we want to rely on len
+	if len(b) == 0 {
+		return ""
+	}
+	// Copied from go 1.20.1 strings.Builder.String
+	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/strings/builder.go#L48
+	return unsafe.String(unsafe.SliceData(b), len(b))
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/envelope.go

@@ -20,120 +20,148 @@ package kmsv2
 import (
 	"context"
 	"crypto/aes"
-	"crypto/cipher"
-	"crypto/rand"
-	"encoding/base64"
 	"fmt"
+	"sort"
 	"time"
+	"unsafe"
 
 	"github.com/gogo/protobuf/proto"
+	"golang.org/x/crypto/cryptobyte"
+
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/storage/value"
-	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1"
+	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
 	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	"k8s.io/klog/v2"
-	"k8s.io/utils/lru"
+	kmsservice "k8s.io/kms/pkg/service"
+	"k8s.io/utils/clock"
 )
 
+func init() {
+	value.RegisterMetrics()
+	metrics.RegisterMetrics()
+}
+
 const (
 	// KMSAPIVersion is the version of the KMS API.
-	KMSAPIVersion = "v2alpha1"
+	KMSAPIVersion = "v2beta1"
 	// annotationsMaxSize is the maximum size of the annotations.
 	annotationsMaxSize = 32 * 1024 // 32 kB
-	// keyIDMaxSize is the maximum size of the keyID.
-	keyIDMaxSize = 1 * 1024 // 1 kB
+	// KeyIDMaxSize is the maximum size of the keyID.
+	KeyIDMaxSize = 1 * 1024 // 1 kB
 	// encryptedDEKMaxSize is the maximum size of the encrypted DEK.
 	encryptedDEKMaxSize = 1 * 1024 // 1 kB
+	// cacheTTL is the default time-to-live for the cache entry.
+	// this allows the cache to grow to an infinite size for up to a day.
+	// this is meant as a temporary solution until the cache is re-written to not have a TTL.
+	// there is unlikely to be any meaningful memory impact on the server
+	// because the cache will likely never have more than a few thousand entries
+	// and each entry is roughly ~200 bytes in size.  with DEK reuse
+	// and no storage migration, the number of entries in this cache
+	// would be approximated by unique key IDs used by the KMS plugin
+	// combined with the number of server restarts.  If storage migration
+	// is performed after key ID changes, and the number of restarts
+	// is limited, this cache size may be as small as the number of API
+	// servers in use (once old entries expire out from the TTL).
+	cacheTTL = 24 * time.Hour
+	// error code
+	errKeyIDOKCode      ErrCodeKeyID = "ok"
+	errKeyIDEmptyCode   ErrCodeKeyID = "empty"
+	errKeyIDTooLongCode ErrCodeKeyID = "too_long"
 )
 
-// Service allows encrypting and decrypting data using an external Key Management Service.
-type Service interface {
-	// Decrypt a given bytearray to obtain the original data as bytes.
-	Decrypt(ctx context.Context, uid string, req *DecryptRequest) ([]byte, error)
-	// Encrypt bytes to a ciphertext.
-	Encrypt(ctx context.Context, uid string, data []byte) (*EncryptResponse, error)
-	// Status returns the status of the KMS.
-	Status(ctx context.Context) (*StatusResponse, error)
+// NowFunc is exported so tests can override it.
+var NowFunc = time.Now
+
+type StateFunc func() (State, error)
+type ErrCodeKeyID string
+
+type State struct {
+	Transformer  value.Transformer
+	EncryptedDEK []byte
+	KeyID        string
+	Annotations  map[string][]byte
+
+	UID string
+
+	ExpirationTimestamp time.Time
+
+	// CacheKey is the key used to cache the DEK in transformer.cache.
+	CacheKey []byte
+}
+
+func (s *State) ValidateEncryptCapability() error {
+	if now := NowFunc(); now.After(s.ExpirationTimestamp) {
+		return fmt.Errorf("EDEK with keyID %q expired at %s (current time is %s)",
+			s.KeyID, s.ExpirationTimestamp.Format(time.RFC3339), now.Format(time.RFC3339))
+	}
+	return nil
 }
 
 type envelopeTransformer struct {
-	envelopeService Service
+	envelopeService kmsservice.Service
+	providerName    string
+	stateFunc       StateFunc
 
-	// transformers is a thread-safe LRU cache which caches decrypted DEKs indexed by their encrypted form.
-	transformers *lru.Cache
-
-	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
-	baseTransformerFunc func(cipher.Block) value.Transformer
-
-	cacheSize    int
-	cacheEnabled bool
-}
-
-// EncryptResponse is the response from the Envelope service when encrypting data.
-type EncryptResponse struct {
-	Ciphertext  []byte
-	KeyID       string
-	Annotations map[string][]byte
-}
-
-// DecryptRequest is the request to the Envelope service when decrypting data.
-type DecryptRequest struct {
-	Ciphertext  []byte
-	KeyID       string
-	Annotations map[string][]byte
-}
-
-// StatusResponse is the response from the Envelope service when getting the status of the service.
-type StatusResponse struct {
-	Version string
-	Healthz string
-	KeyID   string
+	// cache is a thread-safe expiring lru cache which caches decrypted DEKs indexed by their encrypted form.
+	cache *simpleCache
 }
 
 // NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
 // It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
-// the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
-// used decrypted DEKs in memory.
-func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) value.Transformer {
-	var cache *lru.Cache
-
-	if cacheSize > 0 {
-		// TODO(aramase): Switch to using expiring cache: kubernetes/kubernetes/staging/src/k8s.io/apimachinery/pkg/util/cache/expiring.go.
-		// It handles scans a lot better, doesn't have to be right sized, and don't have a global lock on reads.
-		cache = lru.New(cacheSize)
-	}
+// the data items they encrypt.
+func NewEnvelopeTransformer(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc) value.Transformer {
+	return newEnvelopeTransformerWithClock(envelopeService, providerName, stateFunc, cacheTTL, clock.RealClock{})
+}
 
+func newEnvelopeTransformerWithClock(envelopeService kmsservice.Service, providerName string, stateFunc StateFunc, cacheTTL time.Duration, clock clock.Clock) value.Transformer {
 	return &envelopeTransformer{
-		envelopeService:     envelopeService,
-		transformers:        cache,
-		baseTransformerFunc: baseTransformerFunc,
-		cacheEnabled:        cacheSize > 0,
-		cacheSize:           cacheSize,
+		envelopeService: envelopeService,
+		providerName:    providerName,
+		stateFunc:       stateFunc,
+		cache:           newSimpleCache(clock, cacheTTL),
 	}
 }
 
 // TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
 func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
-	metrics.RecordArrival(metrics.FromStorageLabel, time.Now())
-
 	// Deserialize the EncryptedObject from the data.
 	encryptedObject, err := t.doDecode(data)
 	if err != nil {
 		return nil, false, err
 	}
 
-	// Look up the decrypted DEK from cache or Envelope.
-	transformer := t.getTransformer(encryptedObject.EncryptedDEK)
+	// TODO: consider marking state.EncryptedDEK != encryptedObject.EncryptedDEK as a stale read to support DEK defragmentation
+	//  at a minimum we should have a metric that helps the user understand if DEK fragmentation is high
+	state, err := t.stateFunc() // no need to call state.ValidateEncryptCapability on reads
+	if err != nil {
+		return nil, false, err
+	}
+
+	encryptedObjectCacheKey, err := generateCacheKey(encryptedObject.EncryptedDEK, encryptedObject.KeyID, encryptedObject.Annotations)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// Look up the decrypted DEK from cache first
+	transformer := t.cache.get(encryptedObjectCacheKey)
+
+	// fallback to the envelope service if we do not have the transformer locally
 	if transformer == nil {
-		if t.cacheEnabled {
-			value.RecordCacheMiss()
-		}
+		value.RecordCacheMiss()
+
+		requestInfo := getRequestInfoFromContext(ctx)
 		uid := string(uuid.NewUUID())
-		klog.V(6).InfoS("Decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()))
-		key, err := t.envelopeService.Decrypt(ctx, uid, &DecryptRequest{
+		klog.V(6).InfoS("decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()),
+			"group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource,
+			"verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
+
+		key, err := t.envelopeService.Decrypt(ctx, uid, &kmsservice.DecryptRequest{
 			Ciphertext:  encryptedObject.EncryptedDEK,
 			KeyID:       encryptedObject.KeyID,
 			Annotations: encryptedObject.Annotations,
@@ -142,80 +170,79 @@ func (t *envelopeTransformer) TransformFromStorage(ctx context.Context, data []b
 			return nil, false, fmt.Errorf("failed to decrypt DEK, error: %w", err)
 		}
 
-		transformer, err = t.addTransformer(encryptedObject.EncryptedDEK, key)
+		transformer, err = t.addTransformerForDecryption(encryptedObjectCacheKey, key)
 		if err != nil {
 			return nil, false, err
 		}
 	}
+	metrics.RecordKeyID(metrics.FromStorageLabel, t.providerName, encryptedObject.KeyID)
+
+	out, stale, err := transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// data is considered stale if the key ID does not match our current write transformer
+	return out, stale || encryptedObject.KeyID != state.KeyID, nil
 
-	return transformer.TransformFromStorage(ctx, encryptedObject.EncryptedData, dataCtx)
 }
 
 // TransformToStorage encrypts data to be written to disk using envelope encryption.
 func (t *envelopeTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
-	metrics.RecordArrival(metrics.ToStorageLabel, time.Now())
-	newKey, err := generateKey(32)
+	state, err := t.stateFunc()
+	if err != nil {
+		return nil, err
+	}
+	if err := state.ValidateEncryptCapability(); err != nil {
+		return nil, err
+	}
+
+	// this prevents a cache miss every time the DEK rotates
+	// this has the side benefit of causing the cache to perform a GC
+	// TODO see if we can do this inside the stateFunc control loop
+	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
+	t.cache.set(state.CacheKey, state.Transformer)
+
+	requestInfo := getRequestInfoFromContext(ctx)
+	klog.V(6).InfoS("encrypting content using DEK", "uid", state.UID, "key", string(dataCtx.AuthenticatedData()),
+		"group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource,
+		"verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name)
+
+	result, err := state.Transformer.TransformToStorage(ctx, data, dataCtx)
 	if err != nil {
 		return nil, err
 	}
 
-	uid := string(uuid.NewUUID())
-	klog.V(6).InfoS("Encrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()))
-	resp, err := t.envelopeService.Encrypt(ctx, uid, newKey)
-	if err != nil {
-		return nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
-	}
-
-	transformer, err := t.addTransformer(resp.Ciphertext, newKey)
-	if err != nil {
-		return nil, err
-	}
-
-	result, err := transformer.TransformToStorage(ctx, data, dataCtx)
-	if err != nil {
-		return nil, err
-	}
+	metrics.RecordKeyID(metrics.ToStorageLabel, t.providerName, state.KeyID)
 
 	encObject := &kmstypes.EncryptedObject{
-		KeyID:         resp.KeyID,
-		EncryptedDEK:  resp.Ciphertext,
+		KeyID:         state.KeyID,
+		EncryptedDEK:  state.EncryptedDEK,
 		EncryptedData: result,
-		Annotations:   resp.Annotations,
+		Annotations:   state.Annotations,
 	}
 
 	// Serialize the EncryptedObject to a byte array.
 	return t.doEncode(encObject)
 }
 
-// addTransformer inserts a new transformer to the Envelope cache of DEKs for future reads.
-func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.Transformer, error) {
+// addTransformerForDecryption inserts a new transformer to the Envelope cache of DEKs for future reads.
+func (t *envelopeTransformer) addTransformerForDecryption(cacheKey []byte, key []byte) (decryptTransformer, error) {
 	block, err := aes.NewCipher(key)
 	if err != nil {
 		return nil, err
 	}
-	transformer := t.baseTransformerFunc(block)
-	// Use base64 of encKey as the key into the cache because hashicorp/golang-lru
-	// cannot hash []uint8.
-	if t.cacheEnabled {
-		t.transformers.Add(base64.StdEncoding.EncodeToString(encKey), transformer)
-		metrics.RecordDekCacheFillPercent(float64(t.transformers.Len()) / float64(t.cacheSize))
+	// this is compatible with NewGCMTransformerWithUniqueKeyUnsafe for decryption
+	// it would use random nonces for encryption but we never do that
+	transformer, err := aestransformer.NewGCMTransformer(block)
+	if err != nil {
+		return nil, err
 	}
+	// TODO(aramase): Add metrics for cache fill percentage with custom cache implementation.
+	t.cache.set(cacheKey, transformer)
 	return transformer, nil
 }
 
-// getTransformer fetches the transformer corresponding to encKey from cache, if it exists.
-func (t *envelopeTransformer) getTransformer(encKey []byte) value.Transformer {
-	if !t.cacheEnabled {
-		return nil
-	}
-
-	_transformer, found := t.transformers.Get(base64.StdEncoding.EncodeToString(encKey))
-	if found {
-		return _transformer.(value.Transformer)
-	}
-	return nil
-}
-
 // doEncode encodes the EncryptedObject to a byte array.
 func (t *envelopeTransformer) doEncode(request *kmstypes.EncryptedObject) ([]byte, error) {
 	if err := validateEncryptedObject(request); err != nil {
@@ -238,17 +265,34 @@ func (t *envelopeTransformer) doDecode(originalData []byte) (*kmstypes.Encrypted
 	return o, nil
 }
 
-// generateKey generates a random key using system randomness.
-func generateKey(length int) (key []byte, err error) {
-	defer func(start time.Time) {
-		value.RecordDataKeyGeneration(start, err)
-	}(time.Now())
-	key = make([]byte, length)
-	if _, err = rand.Read(key); err != nil {
-		return nil, err
+func GenerateTransformer(ctx context.Context, uid string, envelopeService kmsservice.Service) (value.Transformer, *kmsservice.EncryptResponse, []byte, error) {
+	transformer, newKey, err := aestransformer.NewGCMTransformerWithUniqueKeyUnsafe()
+	if err != nil {
+		return nil, nil, nil, err
 	}
 
-	return key, nil
+	klog.V(6).InfoS("encrypting content using envelope service", "uid", uid)
+
+	resp, err := envelopeService.Encrypt(ctx, uid, newKey)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)
+	}
+
+	if err := validateEncryptedObject(&kmstypes.EncryptedObject{
+		KeyID:         resp.KeyID,
+		EncryptedDEK:  resp.Ciphertext,
+		EncryptedData: []byte{0}, // any non-empty value to pass validation
+		Annotations:   resp.Annotations,
+	}); err != nil {
+		return nil, nil, nil, err
+	}
+
+	cacheKey, err := generateCacheKey(resp.Ciphertext, resp.KeyID, resp.Annotations)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return transformer, resp, cacheKey, nil
 }
 
 func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
@@ -261,7 +305,7 @@ func validateEncryptedObject(o *kmstypes.EncryptedObject) error {
 	if err := validateEncryptedDEK(o.EncryptedDEK); err != nil {
 		return fmt.Errorf("failed to validate encrypted DEK: %w", err)
 	}
-	if err := validateKeyID(o.KeyID); err != nil {
+	if _, err := ValidateKeyID(o.KeyID); err != nil {
 		return fmt.Errorf("failed to validate key id: %w", err)
 	}
 	if err := validateAnnotations(o.Annotations); err != nil {
@@ -301,15 +345,78 @@ func validateAnnotations(annotations map[string][]byte) error {
 	return utilerrors.NewAggregate(errs)
 }
 
-// validateKeyID tests the following:
+// ValidateKeyID tests the following:
 // 1. The keyID is not empty.
 // 2. The size of keyID is less than 1 kB.
-func validateKeyID(keyID string) error {
+func ValidateKeyID(keyID string) (ErrCodeKeyID, error) {
 	if len(keyID) == 0 {
-		return fmt.Errorf("keyID is empty")
+		return errKeyIDEmptyCode, fmt.Errorf("keyID is empty")
 	}
-	if len(keyID) > keyIDMaxSize {
-		return fmt.Errorf("keyID is %d bytes, which exceeds the max size of %d", len(keyID), keyIDMaxSize)
+	if len(keyID) > KeyIDMaxSize {
+		return errKeyIDTooLongCode, fmt.Errorf("keyID is %d bytes, which exceeds the max size of %d", len(keyID), KeyIDMaxSize)
 	}
-	return nil
+	return errKeyIDOKCode, nil
+}
+
+func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestInfo {
+	if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found {
+		return reqInfo
+	}
+	return &genericapirequest.RequestInfo{}
+}
+
+// generateCacheKey returns a key for the cache.
+// The key is a concatenation of:
+//  1. encryptedDEK
+//  2. keyID
+//  3. length of annotations
+//  4. annotations (sorted by key) - each annotation is a concatenation of:
+//     a. annotation key
+//     b. annotation value
+func generateCacheKey(encryptedDEK []byte, keyID string, annotations map[string][]byte) ([]byte, error) {
+	// TODO(aramase): use sync pool buffer to avoid allocations
+	b := cryptobyte.NewBuilder(nil)
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(encryptedDEK)
+	})
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes(toBytes(keyID))
+	})
+	if len(annotations) == 0 {
+		return b.Bytes()
+	}
+
+	// add the length of annotations to the cache key
+	b.AddUint32(uint32(len(annotations)))
+
+	// Sort the annotations by key.
+	keys := make([]string, 0, len(annotations))
+	for k := range annotations {
+		k := k
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		// The maximum size of annotations is annotationsMaxSize (32 kB) so we can safely
+		// assume that the length of the key and value will fit in a uint16.
+		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(toBytes(k))
+		})
+		b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+			b.AddBytes(annotations[k])
+		})
+	}
+
+	return b.Bytes()
+}
+
+// toBytes performs unholy acts to avoid allocations
+func toBytes(s string) []byte {
+	// unsafe.StringData is unspecified for the empty string, so we provide a strict interpretation
+	if len(s) == 0 {
+		return nil
+	}
+	// Copied from go 1.20.1 os.File.WriteString
+	// https://github.com/golang/go/blob/202a1a57064127c3f19d96df57b9f9586145e21c/src/os/file.go#L246
+	return unsafe.Slice(unsafe.StringData(s), len(s))
 }

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/grpc_service.go

@@ -27,9 +27,11 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/util"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
 	"k8s.io/klog/v2"
-	kmsapi "k8s.io/kms/apis/v2alpha1"
+	kmsapi "k8s.io/kms/apis/v2"
+	kmsservice "k8s.io/kms/pkg/service"
+	"k8s.io/kms/pkg/util"
 )
 
 const (
@@ -45,8 +47,8 @@ type gRPCService struct {
 }
 
 // NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider.
-func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Duration) (Service, error) {
-	klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint)
+func NewGRPCService(ctx context.Context, endpoint, providerName string, callTimeout time.Duration) (kmsservice.Service, error) {
+	klog.V(4).InfoS("Configure KMS provider", "endpoint", endpoint)
 
 	addr, err := util.ParseEndpoint(endpoint)
 	if err != nil {
@@ -64,12 +66,14 @@ func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Durat
 				// addr - comes from the closure
 				c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr})
 				if err != nil {
-					klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
+					klog.ErrorS(err, "failed to create connection to unix socket", "addr", addr)
 				} else {
-					klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
+					klog.V(4).InfoS("Successfully dialed Unix socket", "addr", addr)
 				}
 				return c, err
-			}))
+			}),
+		grpc.WithChainUnaryInterceptor(recordMetricsInterceptor(providerName)),
+	)
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to create connection to %s, error: %v", endpoint, err)
@@ -88,7 +92,7 @@ func NewGRPCService(ctx context.Context, endpoint string, callTimeout time.Durat
 }
 
 // Decrypt a given data string to obtain the original byte data.
-func (g *gRPCService) Decrypt(ctx context.Context, uid string, req *DecryptRequest) ([]byte, error) {
+func (g *gRPCService) Decrypt(ctx context.Context, uid string, req *kmsservice.DecryptRequest) ([]byte, error) {
 	ctx, cancel := context.WithTimeout(ctx, g.callTimeout)
 	defer cancel()
 
@@ -106,7 +110,7 @@ func (g *gRPCService) Decrypt(ctx context.Context, uid string, req *DecryptReque
 }
 
 // Encrypt bytes to a string ciphertext.
-func (g *gRPCService) Encrypt(ctx context.Context, uid string, plaintext []byte) (*EncryptResponse, error) {
+func (g *gRPCService) Encrypt(ctx context.Context, uid string, plaintext []byte) (*kmsservice.EncryptResponse, error) {
 	ctx, cancel := context.WithTimeout(ctx, g.callTimeout)
 	defer cancel()
 
@@ -118,7 +122,7 @@ func (g *gRPCService) Encrypt(ctx context.Context, uid string, plaintext []byte)
 	if err != nil {
 		return nil, err
 	}
-	return &EncryptResponse{
+	return &kmsservice.EncryptResponse{
 		Ciphertext:  response.Ciphertext,
 		KeyID:       response.KeyId,
 		Annotations: response.Annotations,
@@ -126,7 +130,7 @@ func (g *gRPCService) Encrypt(ctx context.Context, uid string, plaintext []byte)
 }
 
 // Status returns the status of the KMSv2 provider.
-func (g *gRPCService) Status(ctx context.Context) (*StatusResponse, error) {
+func (g *gRPCService) Status(ctx context.Context) (*kmsservice.StatusResponse, error) {
 	ctx, cancel := context.WithTimeout(ctx, g.callTimeout)
 	defer cancel()
 
@@ -135,5 +139,15 @@ func (g *gRPCService) Status(ctx context.Context) (*StatusResponse, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &StatusResponse{Version: response.Version, Healthz: response.Healthz, KeyID: response.KeyId}, nil
+	return &kmsservice.StatusResponse{Version: response.Version, Healthz: response.Healthz, KeyID: response.KeyId}, nil
+}
+
+func recordMetricsInterceptor(providerName string) grpc.UnaryClientInterceptor {
+	return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {
+		start := NowFunc()
+		respErr := invoker(ctx, method, req, reply, cc, opts...)
+		elapsed := NowFunc().Sub(start)
+		metrics.RecordKMSOperationLatency(providerName, method, elapsed, respErr)
+		return respErr
+	}
 }

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.pb.go

@@ -17,7 +17,7 @@ limitations under the License.
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
 // source: api.proto
 
-package v2alpha1
+package v2
 
 import (
 	fmt "fmt"
@@ -104,25 +104,28 @@ func (m *EncryptedObject) GetAnnotations() map[string][]byte {
 }
 
 func init() {
-	proto.RegisterType((*EncryptedObject)(nil), "v2alpha1.EncryptedObject")
-	proto.RegisterMapType((map[string][]byte)(nil), "v2alpha1.EncryptedObject.AnnotationsEntry")
+	proto.RegisterType((*EncryptedObject)(nil), "v2.EncryptedObject")
+	proto.RegisterMapType((map[string][]byte)(nil), "v2.EncryptedObject.AnnotationsEntry")
 }
 
 func init() { proto.RegisterFile("api.proto", fileDescriptor_00212fb1f9d3bf1c) }
 
 var fileDescriptor_00212fb1f9d3bf1c = []byte{
-	// 200 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4c, 0x2c, 0xc8, 0xd4,
-	0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x28, 0x33, 0x4a, 0xcc, 0x29, 0xc8, 0x48, 0x34, 0x54,
-	0xfa, 0xcf, 0xc8, 0xc5, 0xef, 0x9a, 0x97, 0x5c, 0x54, 0x59, 0x50, 0x92, 0x9a, 0xe2, 0x9f, 0x94,
-	0x95, 0x9a, 0x5c, 0x22, 0xa4, 0xc2, 0xc5, 0x9b, 0x0a, 0x13, 0x72, 0x49, 0x2c, 0x49, 0x94, 0x60,
-	0x54, 0x60, 0xd4, 0xe0, 0x09, 0x42, 0x15, 0x14, 0x12, 0xe1, 0x62, 0xcd, 0x4e, 0xad, 0xf4, 0x74,
-	0x91, 0x60, 0x52, 0x60, 0xd4, 0xe0, 0x0c, 0x82, 0x70, 0x84, 0x94, 0xb8, 0x78, 0x10, 0xca, 0x5c,
-	0xbd, 0x25, 0x98, 0xc1, 0x5a, 0x51, 0xc4, 0x84, 0x7c, 0xb8, 0xb8, 0x13, 0xf3, 0xf2, 0xf2, 0x4b,
-	0x12, 0x4b, 0x32, 0xf3, 0xf3, 0x8a, 0x25, 0x58, 0x14, 0x98, 0x35, 0xb8, 0x8d, 0xb4, 0xf4, 0x60,
-	0x6e, 0xd2, 0x43, 0x73, 0x8f, 0x9e, 0x23, 0x42, 0xb1, 0x6b, 0x5e, 0x49, 0x51, 0x65, 0x10, 0xb2,
-	0x76, 0x29, 0x3b, 0x2e, 0x01, 0x74, 0x05, 0x42, 0x02, 0x5c, 0xcc, 0xd9, 0xa9, 0x95, 0x60, 0x77,
-	0x73, 0x06, 0x81, 0x98, 0x20, 0xd7, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x82, 0x5d, 0xcb, 0x13, 0x04,
-	0xe1, 0x58, 0x31, 0x59, 0x30, 0x26, 0xb1, 0x81, 0x83, 0xc4, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff,
-	0x88, 0x8c, 0xbb, 0x4e, 0x1f, 0x01, 0x00, 0x00,
+	// 244 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x90, 0xb1, 0x4b, 0x03, 0x31,
+	0x14, 0xc6, 0xc9, 0x9d, 0x0a, 0x97, 0x9e, 0x58, 0x82, 0xc3, 0xe1, 0x74, 0x94, 0x0e, 0x37, 0x25,
+	0x10, 0x97, 0x22, 0x52, 0x50, 0x7a, 0x82, 0x38, 0x08, 0x19, 0xdd, 0xd2, 0xfa, 0x28, 0x67, 0x6a,
+	0x12, 0x92, 0x18, 0xc8, 0x9f, 0xee, 0x26, 0x4d, 0x95, 0xda, 0xdb, 0xde, 0xf7, 0xf1, 0xfb, 0xe0,
+	0xc7, 0xc3, 0x95, 0xb4, 0x03, 0xb5, 0xce, 0x04, 0x43, 0x8a, 0xc8, 0x67, 0xdf, 0x08, 0x5f, 0xf5,
+	0x7a, 0xe3, 0x92, 0x0d, 0xf0, 0xfe, 0xba, 0xfe, 0x80, 0x4d, 0x20, 0x73, 0x7c, 0x09, 0x7f, 0xd5,
+	0x4a, 0x06, 0xd9, 0xa0, 0x16, 0x75, 0xb5, 0x38, 0x2d, 0xc9, 0x35, 0x3e, 0x57, 0x90, 0x9e, 0x57,
+	0x4d, 0xd1, 0xa2, 0xae, 0x12, 0x87, 0x40, 0x66, 0xb8, 0x3e, 0x62, 0xfd, 0x4b, 0x53, 0xe6, 0xe9,
+	0x49, 0x47, 0x9e, 0xf0, 0x44, 0x6a, 0x6d, 0x82, 0x0c, 0x83, 0xd1, 0xbe, 0x39, 0x6b, 0xcb, 0x6e,
+	0xc2, 0xe7, 0x34, 0x72, 0x3a, 0x32, 0xa1, 0x0f, 0x47, 0xac, 0xd7, 0xc1, 0x25, 0xf1, 0x7f, 0x78,
+	0xb3, 0xc4, 0xd3, 0x31, 0x40, 0xa6, 0xb8, 0x54, 0x90, 0xb2, 0x71, 0x25, 0xf6, 0xe7, 0xde, 0x33,
+	0xca, 0xdd, 0x17, 0x64, 0xcf, 0x5a, 0x1c, 0xc2, 0x5d, 0xb1, 0x40, 0x8f, 0xcb, 0xb7, 0x7b, 0xb5,
+	0xf0, 0x74, 0x30, 0x4c, 0xda, 0xc1, 0x83, 0x8b, 0xe0, 0x98, 0x55, 0x5b, 0xe6, 0x83, 0x71, 0x72,
+	0x0b, 0x2c, 0x93, 0xec, 0x57, 0x9d, 0x81, 0x8e, 0xb0, 0x33, 0x16, 0x98, 0xfa, 0xf4, 0x91, 0xb3,
+	0xc8, 0xd7, 0x17, 0xf9, 0x8d, 0xb7, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x80, 0x43, 0x93,
+	0x53, 0x01, 0x00, 0x00,
 }

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/api.proto

@@ -14,10 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// To regenerate api.pb.go run hack/update-generated-kms.sh
+// To regenerate api.pb.go run `hack/update-codegen.sh protobindings`
 syntax = "proto3";
 
-package v2alpha1;
+package v2;
+option go_package = "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2";
 
 // EncryptedObject is the representation of data stored in etcd after envelope encryption.
 message EncryptedObject {

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2/v2.go

@@ -14,5 +14,5 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package v2alpha1 contains definition of kms-plugin's serialized types.
-package v2alpha1
+// Package v2 contains definition of kms-plugin's serialized types.
+package v2

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics/metrics.go

@@ -17,11 +17,20 @@ limitations under the License.
 package metrics
 
 import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"hash"
 	"sync"
 	"time"
 
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+
 	"k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
+	"k8s.io/utils/lru"
 )
 
 const (
@@ -31,6 +40,12 @@ const (
 	ToStorageLabel   = "to_storage"
 )
 
+type metricLabels struct {
+	transformationType string
+	providerName       string
+	keyIDHash          string
+}
+
 /*
  * By default, all the following metrics are defined as falling under
  * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/1209-metrics-stability/kubernetes-control-plane-metrics-stability.md#stability-classes)
@@ -40,12 +55,18 @@ const (
  * the metric stability policy.
  */
 var (
-	lockLastFromStorage sync.Mutex
-	lockLastToStorage   sync.Mutex
+	lockLastFromStorage   sync.Mutex
+	lockLastToStorage     sync.Mutex
+	lockRecordKeyID       sync.Mutex
+	lockRecordKeyIDStatus sync.Mutex
 
-	lastFromStorage time.Time
-	lastToStorage   time.Time
+	lastFromStorage                                 time.Time
+	lastToStorage                                   time.Time
+	keyIDHashTotalMetricLabels                      *lru.Cache
+	keyIDHashStatusLastTimestampSecondsMetricLabels *lru.Cache
+	cacheSize                                       = 100
 
+	// This metric is only used for KMS v1 API.
 	dekCacheFillPercent = metrics.NewGauge(
 		&metrics.GaugeOpts{
 			Namespace:      namespace,
@@ -56,6 +77,7 @@ var (
 		},
 	)
 
+	// This metric is only used for KMS v1 API.
 	dekCacheInterArrivals = metrics.NewHistogramVec(
 		&metrics.HistogramOpts{
 			Namespace:      namespace,
@@ -67,17 +89,145 @@ var (
 		},
 		[]string{"transformation_type"},
 	)
+
+	// These metrics are made public to be used by unit tests.
+	KMSOperationsLatencyMetric = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "kms_operations_latency_seconds",
+			Help:           "KMS operation duration with gRPC error code status total.",
+			StabilityLevel: metrics.ALPHA,
+			// Use custom buckets to avoid the default buckets which are too small for KMS operations.
+			// Start 0.1ms with the last bucket being [~52s, +Inf)
+			Buckets: metrics.ExponentialBuckets(0.0001, 2, 20),
+		},
+		[]string{"provider_name", "method_name", "grpc_status_code"},
+	)
+
+	// keyIDHashTotal is the number of times a keyID is used
+	// e.g. apiserver_envelope_encryption_key_id_hash_total counter
+	// apiserver_envelope_encryption_key_id_hash_total{key_id_hash="sha256",
+	// provider_name="providerName",transformation_type="from_storage"} 1
+	KeyIDHashTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "key_id_hash_total",
+			Help:           "Number of times a keyID is used split by transformation type and provider.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"transformation_type", "provider_name", "key_id_hash"},
+	)
+
+	// keyIDHashLastTimestampSeconds is the last time in seconds when a keyID was used
+	// e.g. apiserver_envelope_encryption_key_id_hash_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName",transformation_type="from_storage"} 1.674865558833728e+09
+	KeyIDHashLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "key_id_hash_last_timestamp_seconds",
+			Help:           "The last time in seconds when a keyID was used.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"transformation_type", "provider_name", "key_id_hash"},
+	)
+
+	// keyIDHashStatusLastTimestampSeconds is the last time in seconds when a keyID was returned by the Status RPC call.
+	// e.g. apiserver_envelope_encryption_key_id_hash_status_last_timestamp_seconds{key_id_hash="sha256", provider_name="providerName"} 1.674865558833728e+09
+	KeyIDHashStatusLastTimestampSeconds = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "key_id_hash_status_last_timestamp_seconds",
+			Help:           "The last time in seconds when a keyID was returned by the Status RPC call.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"provider_name", "key_id_hash"},
+	)
+
+	InvalidKeyIDFromStatusTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "invalid_key_id_from_status_total",
+			Help:           "Number of times an invalid keyID is returned by the Status RPC call split by error.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"provider_name", "error"},
+	)
 )
 
 var registerMetricsFunc sync.Once
+var hashPool *sync.Pool
 
+func registerLRUMetrics() {
+	if keyIDHashTotalMetricLabels != nil {
+		keyIDHashTotalMetricLabels.Clear()
+	}
+	if keyIDHashStatusLastTimestampSecondsMetricLabels != nil {
+		keyIDHashStatusLastTimestampSecondsMetricLabels.Clear()
+	}
+
+	keyIDHashTotalMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
+		item := key.(metricLabels)
+		if deleted := KeyIDHashTotal.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
+			klog.InfoS("Deleted keyIDHashTotalMetricLabels", "transformationType", item.transformationType,
+				"providerName", item.providerName, "keyIDHash", item.keyIDHash)
+		}
+		if deleted := KeyIDHashLastTimestampSeconds.DeleteLabelValues(item.transformationType, item.providerName, item.keyIDHash); deleted {
+			klog.InfoS("Deleted keyIDHashLastTimestampSecondsMetricLabels", "transformationType", item.transformationType,
+				"providerName", item.providerName, "keyIDHash", item.keyIDHash)
+		}
+	})
+	keyIDHashStatusLastTimestampSecondsMetricLabels = lru.NewWithEvictionFunc(cacheSize, func(key lru.Key, _ interface{}) {
+		item := key.(metricLabels)
+		if deleted := KeyIDHashStatusLastTimestampSeconds.DeleteLabelValues(item.providerName, item.keyIDHash); deleted {
+			klog.InfoS("Deleted keyIDHashStatusLastTimestampSecondsMetricLabels", "providerName", item.providerName, "keyIDHash", item.keyIDHash)
+		}
+	})
+}
 func RegisterMetrics() {
 	registerMetricsFunc.Do(func() {
+		registerLRUMetrics()
+		hashPool = &sync.Pool{
+			New: func() interface{} {
+				return sha256.New()
+			},
+		}
 		legacyregistry.MustRegister(dekCacheFillPercent)
 		legacyregistry.MustRegister(dekCacheInterArrivals)
+		legacyregistry.MustRegister(KeyIDHashTotal)
+		legacyregistry.MustRegister(KeyIDHashLastTimestampSeconds)
+		legacyregistry.MustRegister(KeyIDHashStatusLastTimestampSeconds)
+		legacyregistry.MustRegister(InvalidKeyIDFromStatusTotal)
+		legacyregistry.MustRegister(KMSOperationsLatencyMetric)
 	})
 }
 
+// RecordKeyID records total count and last time in seconds when a KeyID was used for TransformFromStorage and TransformToStorage operations
+func RecordKeyID(transformationType, providerName, keyID string) {
+	lockRecordKeyID.Lock()
+	defer lockRecordKeyID.Unlock()
+
+	keyIDHash := addLabelToCache(keyIDHashTotalMetricLabels, transformationType, providerName, keyID)
+	KeyIDHashTotal.WithLabelValues(transformationType, providerName, keyIDHash).Inc()
+	KeyIDHashLastTimestampSeconds.WithLabelValues(transformationType, providerName, keyIDHash).SetToCurrentTime()
+}
+
+// RecordKeyIDFromStatus records last time in seconds when a KeyID was returned by the Status RPC call.
+func RecordKeyIDFromStatus(providerName, keyID string) {
+	lockRecordKeyIDStatus.Lock()
+	defer lockRecordKeyIDStatus.Unlock()
+
+	keyIDHash := addLabelToCache(keyIDHashStatusLastTimestampSecondsMetricLabels, "", providerName, keyID)
+	KeyIDHashStatusLastTimestampSeconds.WithLabelValues(providerName, keyIDHash).SetToCurrentTime()
+}
+
+func RecordInvalidKeyIDFromStatus(providerName, errCode string) {
+	InvalidKeyIDFromStatusTotal.WithLabelValues(providerName, errCode).Inc()
+}
+
 func RecordArrival(transformationType string, start time.Time) {
 	switch transformationType {
 	case FromStorageLabel:
@@ -104,3 +254,51 @@ func RecordArrival(transformationType string, start time.Time) {
 func RecordDekCacheFillPercent(percent float64) {
 	dekCacheFillPercent.Set(percent)
 }
+
+// RecordKMSOperationLatency records the latency of KMS operation.
+func RecordKMSOperationLatency(providerName, methodName string, duration time.Duration, err error) {
+	KMSOperationsLatencyMetric.WithLabelValues(providerName, methodName, getErrorCode(err)).Observe(duration.Seconds())
+}
+
+type gRPCError interface {
+	GRPCStatus() *status.Status
+}
+
+func getErrorCode(err error) string {
+	if err == nil {
+		return codes.OK.String()
+	}
+
+	// handle errors wrapped with fmt.Errorf and similar
+	var s gRPCError
+	if errors.As(err, &s) {
+		return s.GRPCStatus().Code().String()
+	}
+
+	// This is not gRPC error. The operation must have failed before gRPC
+	// method was called, otherwise we would get gRPC error.
+	return "unknown-non-grpc"
+}
+
+func getHash(data string) string {
+	h := hashPool.Get().(hash.Hash)
+	h.Reset()
+	h.Write([]byte(data))
+	result := fmt.Sprintf("sha256:%x", h.Sum(nil))
+	hashPool.Put(h)
+	return result
+}
+
+func addLabelToCache(c *lru.Cache, transformationType, providerName, keyID string) string {
+	keyIDHash := ""
+	// only get hash if the keyID is not empty
+	if len(keyID) > 0 {
+		keyIDHash = getHash(keyID)
+	}
+	c.Add(metricLabels{
+		transformationType: transformationType,
+		providerName:       providerName,
+		keyIDHash:          keyIDHash,
+	}, nil) // value is irrelevant, this is a set and not a map
+	return keyIDHash
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/util/util.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2022 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"fmt"
-	"net/url"
-	"strings"
-)
-
-const (
-	// unixProtocol is the only supported protocol for remote KMS provider.
-	unixProtocol = "unix"
-)
-
-// Parse the endpoint to extract schema, host or path.
-func ParseEndpoint(endpoint string) (string, error) {
-	if len(endpoint) == 0 {
-		return "", fmt.Errorf("remote KMS provider can't use empty string as endpoint")
-	}
-
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return "", fmt.Errorf("invalid endpoint %q for remote KMS provider, error: %v", endpoint, err)
-	}
-
-	if u.Scheme != unixProtocol {
-		return "", fmt.Errorf("unsupported scheme %q for remote KMS provider", u.Scheme)
-	}
-
-	// Linux abstract namespace socket - no physical file required
-	// Warning: Linux Abstract sockets have not concept of ACL (unlike traditional file based sockets).
-	// However, Linux Abstract sockets are subject to Linux networking namespace, so will only be accessible to
-	// containers within the same pod (unless host networking is used).
-	if strings.HasPrefix(u.Path, "/@") {
-		return strings.TrimPrefix(u.Path, "/"), nil
-	}
-
-	return u.Path, nil
-}

vendor/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -51,7 +51,7 @@ var (
 			Buckets:        metrics.ExponentialBuckets(5e-6, 2, 25),
 			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"transformation_type"},
+		[]string{"transformation_type", "transformer_prefix"},
 	)
 
 	transformerOperationsTotal = metrics.NewCounterVec(
@@ -111,12 +111,11 @@ func RegisterMetrics() {
 
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
 // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
-func RecordTransformation(transformationType, transformerPrefix string, start time.Time, err error) {
+func RecordTransformation(transformationType, transformerPrefix string, elapsed time.Duration, err error) {
 	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, status.Code(err).String()).Inc()
 
-	switch {
-	case err == nil:
-		transformerLatencies.WithLabelValues(transformationType).Observe(sinceInSeconds(start))
+	if err == nil {
+		transformerLatencies.WithLabelValues(transformationType, transformerPrefix).Observe(elapsed.Seconds())
 	}
 }
 

vendor/k8s.io/apiserver/pkg/storage/value/transformer.go

@@ -100,9 +100,9 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
 				continue
 			}
 			if len(transformer.Prefix) == 0 {
-				RecordTransformation("from_storage", "identity", start, err)
+				RecordTransformation("from_storage", "identity", time.Since(start), err)
 			} else {
-				RecordTransformation("from_storage", string(transformer.Prefix), start, err)
+				RecordTransformation("from_storage", string(transformer.Prefix), time.Since(start), err)
 			}
 
 			// It is valid to have overlapping prefixes when the same encryption provider
@@ -146,7 +146,7 @@ func (t *prefixTransformers) TransformFromStorage(ctx context.Context, data []by
 	if err := errors.Reduce(errors.NewAggregate(errs)); err != nil {
 		return nil, false, err
 	}
-	RecordTransformation("from_storage", "unknown", start, t.err)
+	RecordTransformation("from_storage", "unknown", time.Since(start), t.err)
 	return nil, false, t.err
 }
 
@@ -155,7 +155,7 @@ func (t *prefixTransformers) TransformToStorage(ctx context.Context, data []byte
 	start := time.Now()
 	transformer := t.transformers[0]
 	result, err := transformer.Transformer.TransformToStorage(ctx, data, dataCtx)
-	RecordTransformation("to_storage", string(transformer.Prefix), start, err)
+	RecordTransformation("to_storage", string(transformer.Prefix), time.Since(start), err)
 	if err != nil {
 		return nil, err
 	}