mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 18:43:34 +00:00
rebase: update kubernetes to 1.28.0 in main
updating kubernetes to 1.28.0 in the main repo. Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna
committed by
mergify[bot]
mergify[bot]
parent
b2fdc269c3
commit
ff3e84ad67
284
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
generated
vendored
284
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go
generated
vendored
@ -43,10 +43,11 @@ import (
|
||||
"k8s.io/apiserver/pkg/apis/config/validation"
|
||||
"k8s.io/apiserver/pkg/features"
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/storage/value"
|
||||
storagevalue "k8s.io/apiserver/pkg/storage/value"
|
||||
aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
|
||||
envelopekmsv2 "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2"
|
||||
kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
|
||||
"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
|
||||
@ -63,13 +64,13 @@ const (
|
||||
kmsTransformerPrefixV2 = "k8s:enc:kms:v2:"
|
||||
|
||||
// these constants relate to how the KMS v2 plugin status poll logic
|
||||
// and the DEK generation logic behave. In particular, the positive
|
||||
// and the DEK/seed generation logic behave. In particular, the positive
|
||||
// interval and max TTL are closely related as the difference between
|
||||
// these values defines the worst case window in which the write DEK
|
||||
// these values defines the worst case window in which the write DEK/seed
|
||||
// could expire due to the plugin going into an error state. The
|
||||
// worst case window divided by the negative interval defines the
|
||||
// minimum amount of times the server will attempt to return to a
|
||||
// healthy state before the DEK expires and writes begin to fail.
|
||||
// healthy state before the DEK/seed expires and writes begin to fail.
|
||||
//
|
||||
// For now, these values are kept small and hardcoded to support being
|
||||
// able to perform a "passive" storage migration while tolerating some
|
||||
@ -82,13 +83,13 @@ const (
|
||||
// At that point, they are guaranteed to either migrate to the new key
|
||||
// or get errors during the migration.
|
||||
//
|
||||
// If the API server coasted forever on the last DEK, they would need
|
||||
// If the API server coasted forever on the last DEK/seed, they would need
|
||||
// to actively check if it had observed the new key ID before starting
|
||||
// a migration - otherwise it could keep using the old DEK and their
|
||||
// a migration - otherwise it could keep using the old DEK/seed and their
|
||||
// storage migration would not do what they thought it did.
|
||||
kmsv2PluginHealthzPositiveInterval = 1 * time.Minute
|
||||
kmsv2PluginHealthzNegativeInterval = 10 * time.Second
|
||||
kmsv2PluginWriteDEKMaxTTL = 3 * time.Minute
|
||||
kmsv2PluginWriteDEKSourceMaxTTL = 3 * time.Minute
|
||||
|
||||
kmsPluginHealthzNegativeTTL = 3 * time.Second
|
||||
kmsPluginHealthzPositiveTTL = 20 * time.Second
|
||||
@ -159,7 +160,7 @@ func (h *kmsv2PluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
|
||||
// EncryptionConfiguration represents the parsed and normalized encryption configuration for the apiserver.
|
||||
type EncryptionConfiguration struct {
|
||||
// Transformers is a list of value.Transformer that will be used to encrypt and decrypt data.
|
||||
Transformers map[schema.GroupResource]value.Transformer
|
||||
Transformers map[schema.GroupResource]storagevalue.Transformer
|
||||
|
||||
// HealthChecks is a list of healthz.HealthChecker that will be used to check the health of the encryption providers.
|
||||
HealthChecks []healthz.HealthChecker
|
||||
@ -207,7 +208,7 @@ func LoadEncryptionConfig(ctx context.Context, filepath string, reload bool) (*E
|
||||
// getTransformerOverridesAndKMSPluginHealthzCheckers creates the set of transformers and KMS healthz checks based on the given config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]value.Transformer, []healthz.HealthChecker, *kmsState, error) {
|
||||
func getTransformerOverridesAndKMSPluginHealthzCheckers(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthz.HealthChecker, *kmsState, error) {
|
||||
var kmsHealthChecks []healthz.HealthChecker
|
||||
transformers, probes, kmsUsed, err := getTransformerOverridesAndKMSPluginProbes(ctx, config)
|
||||
if err != nil {
|
||||
@ -228,8 +229,8 @@ type healthChecker interface {
|
||||
// getTransformerOverridesAndKMSPluginProbes creates the set of transformers and KMS probes based on the given config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]value.Transformer, []healthChecker, *kmsState, error) {
|
||||
resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
|
||||
func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apiserverconfig.EncryptionConfiguration) (map[schema.GroupResource]storagevalue.Transformer, []healthChecker, *kmsState, error) {
|
||||
resourceToPrefixTransformer := map[schema.GroupResource][]storagevalue.PrefixTransformer{}
|
||||
var probes []healthChecker
|
||||
var kmsUsed kmsState
|
||||
|
||||
@ -268,11 +269,11 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis
|
||||
probes = append(probes, p...)
|
||||
}
|
||||
|
||||
transformers := make(map[schema.GroupResource]value.Transformer, len(resourceToPrefixTransformer))
|
||||
transformers := make(map[schema.GroupResource]storagevalue.Transformer, len(resourceToPrefixTransformer))
|
||||
for gr, transList := range resourceToPrefixTransformer {
|
||||
gr := gr
|
||||
transList := transList
|
||||
transformers[gr] = value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
|
||||
transformers[gr] = storagevalue.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...)
|
||||
}
|
||||
|
||||
return transformers, probes, &kmsUsed, nil
|
||||
@ -332,8 +333,8 @@ func (h *kmsv2PluginProbe) check(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// rotateDEKOnKeyIDChange tries to rotate to a new DEK if the key ID returned by Status does not match the
|
||||
// current state. If a successful rotation is performed, the new DEK and keyID overwrite the existing state.
|
||||
// rotateDEKOnKeyIDChange tries to rotate to a new DEK/seed if the key ID returned by Status does not match the
|
||||
// current state. If a successful rotation is performed, the new DEK/seed and keyID overwrite the existing state.
|
||||
// On any failure during rotation (including mismatch between status and encrypt calls), the current state is
|
||||
// preserved and will remain valid to use for encryption until its expiration (the system attempts to coast).
|
||||
// If the key ID returned by Status matches the current state, the expiration of the current state is extended
|
||||
@ -346,47 +347,62 @@ func (h *kmsv2PluginProbe) rotateDEKOnKeyIDChange(ctx context.Context, statusKey
|
||||
|
||||
// allow reads indefinitely in all cases
|
||||
// allow writes indefinitely as long as there is no error
|
||||
// allow writes for only up to kmsv2PluginWriteDEKMaxTTL from now when there are errors
|
||||
// we start the timer before we make the network call because kmsv2PluginWriteDEKMaxTTL is meant to be the upper bound
|
||||
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKMaxTTL)
|
||||
// allow writes for only up to kmsv2PluginWriteDEKSourceMaxTTL from now when there are errors
|
||||
// we start the timer before we make the network call because kmsv2PluginWriteDEKSourceMaxTTL is meant to be the upper bound
|
||||
expirationTimestamp := envelopekmsv2.NowFunc().Add(kmsv2PluginWriteDEKSourceMaxTTL)
|
||||
|
||||
// state is valid and status keyID is unchanged from when we generated this DEK so there is no need to rotate it
|
||||
// dynamically check if we want to use KDF seed to derive DEKs or just a single DEK
|
||||
// this gate can only change during tests, but the check is cheap enough to always make
|
||||
// this allows us to easily exercise both modes without restarting the API server
|
||||
// TODO integration test that this dynamically takes effect
|
||||
useSeed := utilfeature.DefaultFeatureGate.Enabled(features.KMSv2KDF)
|
||||
stateUseSeed := state.EncryptedObject.EncryptedDEKSourceType == kmstypes.EncryptedDEKSourceType_HKDF_SHA256_XNONCE_AES_GCM_SEED
|
||||
|
||||
// state is valid and status keyID is unchanged from when we generated this DEK/seed so there is no need to rotate it
|
||||
// just move the expiration of the current state forward by the reuse interval
|
||||
if errState == nil && state.KeyID == statusKeyID {
|
||||
// useSeed can only change at runtime during tests, so we check it here to allow us to easily exercise both modes
|
||||
if errState == nil && state.EncryptedObject.KeyID == statusKeyID && stateUseSeed == useSeed {
|
||||
state.ExpirationTimestamp = expirationTimestamp
|
||||
h.state.Store(&state)
|
||||
return nil
|
||||
}
|
||||
|
||||
transformer, resp, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service)
|
||||
transformer, encObject, cacheKey, errGen := envelopekmsv2.GenerateTransformer(ctx, uid, h.service, useSeed)
|
||||
|
||||
if resp == nil {
|
||||
resp = &kmsservice.EncryptResponse{} // avoid nil panics
|
||||
if encObject == nil {
|
||||
encObject = &kmstypes.EncryptedObject{} // avoid nil panics
|
||||
}
|
||||
|
||||
// happy path, should be the common case
|
||||
// TODO maybe add success metrics?
|
||||
if errGen == nil && resp.KeyID == statusKeyID {
|
||||
if errGen == nil && encObject.KeyID == statusKeyID {
|
||||
h.state.Store(&envelopekmsv2.State{
|
||||
Transformer: transformer,
|
||||
EncryptedDEK: resp.Ciphertext,
|
||||
KeyID: resp.KeyID,
|
||||
Annotations: resp.Annotations,
|
||||
EncryptedObject: *encObject,
|
||||
UID: uid,
|
||||
ExpirationTimestamp: expirationTimestamp,
|
||||
CacheKey: cacheKey,
|
||||
})
|
||||
klog.V(6).InfoS("successfully rotated DEK",
|
||||
"uid", uid,
|
||||
"newKeyID", resp.KeyID,
|
||||
"oldKeyID", state.KeyID,
|
||||
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
|
||||
)
|
||||
return nil
|
||||
|
||||
// it should be logically impossible for the new state to be invalid but check just in case
|
||||
_, errGen = h.getCurrentState()
|
||||
if errGen == nil {
|
||||
klogV6 := klog.V(6)
|
||||
if klogV6.Enabled() {
|
||||
klogV6.InfoS("successfully rotated DEK",
|
||||
"uid", uid,
|
||||
"useSeed", useSeed,
|
||||
"newKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID),
|
||||
"oldKeyIDHash", envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID),
|
||||
"expirationTimestamp", expirationTimestamp.Format(time.RFC3339),
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed to rotate DEK uid=%q, errState=%v, errGen=%v, statusKeyID=%q, encryptKeyID=%q, stateKeyID=%q, expirationTimestamp=%s",
|
||||
uid, errState, errGen, statusKeyID, resp.KeyID, state.KeyID, state.ExpirationTimestamp.Format(time.RFC3339))
|
||||
return fmt.Errorf("failed to rotate DEK uid=%q, useSeed=%v, errState=%v, errGen=%v, statusKeyIDHash=%q, encryptKeyIDHash=%q, stateKeyIDHash=%q, expirationTimestamp=%s",
|
||||
uid, useSeed, errState, errGen, envelopekmsv2.GetHashIfNotEmpty(statusKeyID), envelopekmsv2.GetHashIfNotEmpty(encObject.KeyID), envelopekmsv2.GetHashIfNotEmpty(state.EncryptedObject.KeyID), state.ExpirationTimestamp.Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// getCurrentState returns the latest state from the last status and encrypt calls.
|
||||
@ -399,12 +415,13 @@ func (h *kmsv2PluginProbe) getCurrentState() (envelopekmsv2.State, error) {
|
||||
return envelopekmsv2.State{}, fmt.Errorf("got unexpected nil transformer")
|
||||
}
|
||||
|
||||
if len(state.EncryptedDEK) == 0 {
|
||||
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty EncryptedDEK")
|
||||
encryptedObjectCopy := state.EncryptedObject
|
||||
if len(encryptedObjectCopy.EncryptedData) != 0 {
|
||||
return envelopekmsv2.State{}, fmt.Errorf("got unexpected non-empty EncryptedData")
|
||||
}
|
||||
|
||||
if len(state.KeyID) == 0 {
|
||||
return envelopekmsv2.State{}, fmt.Errorf("got unexpected empty keyID")
|
||||
encryptedObjectCopy.EncryptedData = []byte{0} // any non-empty value to pass validation
|
||||
if err := envelopekmsv2.ValidateEncryptedObject(&encryptedObjectCopy); err != nil {
|
||||
return envelopekmsv2.State{}, fmt.Errorf("got invalid EncryptedObject: %w", err)
|
||||
}
|
||||
|
||||
if state.ExpirationTimestamp.IsZero() {
|
||||
@ -429,7 +446,7 @@ func (h *kmsv2PluginProbe) isKMSv2ProviderHealthyAndMaybeRotateDEK(ctx context.C
|
||||
|
||||
if errCode, err := envelopekmsv2.ValidateKeyID(response.KeyID); err != nil {
|
||||
metrics.RecordInvalidKeyIDFromStatus(h.name, string(errCode))
|
||||
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID %q: %w", response.KeyID, err))
|
||||
errs = append(errs, fmt.Errorf("got invalid KMSv2 KeyID hash %q: %w", envelopekmsv2.GetHashIfNotEmpty(response.KeyID), err))
|
||||
} else {
|
||||
metrics.RecordKeyIDFromStatus(h.name, response.KeyID)
|
||||
// unconditionally append as we filter out nil errors below
|
||||
@ -478,15 +495,15 @@ func loadConfig(filepath string, reload bool) (*apiserverconfig.EncryptionConfig
|
||||
// prefixTransformersAndProbes creates the set of transformers and KMS probes based on the given resource config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, []healthChecker, *kmsState, error) {
|
||||
var transformers []value.PrefixTransformer
|
||||
func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.ResourceConfiguration) ([]storagevalue.PrefixTransformer, []healthChecker, *kmsState, error) {
|
||||
var transformers []storagevalue.PrefixTransformer
|
||||
var probes []healthChecker
|
||||
var kmsUsed kmsState
|
||||
|
||||
for _, provider := range config.Providers {
|
||||
provider := provider
|
||||
var (
|
||||
transformer value.PrefixTransformer
|
||||
transformer storagevalue.PrefixTransformer
|
||||
transformerErr error
|
||||
probe healthChecker
|
||||
used *kmsState
|
||||
@ -497,7 +514,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
|
||||
transformer, transformerErr = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
|
||||
|
||||
case provider.AESCBC != nil:
|
||||
cbcTransformer := func(block cipher.Block) (value.Transformer, error) {
|
||||
cbcTransformer := func(block cipher.Block) (storagevalue.Transformer, error) {
|
||||
return aestransformer.NewCBCTransformer(block), nil
|
||||
}
|
||||
transformer, transformerErr = aesPrefixTransformer(provider.AESCBC, cbcTransformer, aesCBCTransformerPrefixV1)
|
||||
@ -513,7 +530,7 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
|
||||
}
|
||||
|
||||
case provider.Identity != nil:
|
||||
transformer = value.PrefixTransformer{
|
||||
transformer = storagevalue.PrefixTransformer{
|
||||
Transformer: identity.NewEncryptCheckTransformer(),
|
||||
Prefix: []byte{},
|
||||
}
|
||||
@ -532,10 +549,10 @@ func prefixTransformersAndProbes(ctx context.Context, config apiserverconfig.Res
|
||||
return transformers, probes, &kmsUsed, nil
|
||||
}
|
||||
|
||||
type blockTransformerFunc func(cipher.Block) (value.Transformer, error)
|
||||
type blockTransformerFunc func(cipher.Block) (storagevalue.Transformer, error)
|
||||
|
||||
func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
|
||||
var result value.PrefixTransformer
|
||||
func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (storagevalue.PrefixTransformer, error) {
|
||||
var result storagevalue.PrefixTransformer
|
||||
|
||||
if len(config.Keys) == 0 {
|
||||
return result, fmt.Errorf("aes provider has no valid keys")
|
||||
@ -550,7 +567,7 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
|
||||
}
|
||||
}
|
||||
|
||||
keyTransformers := []value.PrefixTransformer{}
|
||||
keyTransformers := []storagevalue.PrefixTransformer{}
|
||||
|
||||
for _, keyData := range config.Keys {
|
||||
keyData := keyData
|
||||
@ -569,26 +586,26 @@ func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTran
|
||||
|
||||
// Create a new PrefixTransformer for this key
|
||||
keyTransformers = append(keyTransformers,
|
||||
value.PrefixTransformer{
|
||||
storagevalue.PrefixTransformer{
|
||||
Transformer: transformer,
|
||||
Prefix: []byte(keyData.Name + ":"),
|
||||
})
|
||||
}
|
||||
|
||||
// Create a prefixTransformer which can choose between these keys
|
||||
keyTransformer := value.NewPrefixTransformers(
|
||||
keyTransformer := storagevalue.NewPrefixTransformers(
|
||||
fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
|
||||
|
||||
// Create a PrefixTransformer which shall later be put in a list with other providers
|
||||
result = value.PrefixTransformer{
|
||||
result = storagevalue.PrefixTransformer{
|
||||
Transformer: keyTransformer,
|
||||
Prefix: []byte(prefix),
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
|
||||
var result value.PrefixTransformer
|
||||
func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (storagevalue.PrefixTransformer, error) {
|
||||
var result storagevalue.PrefixTransformer
|
||||
|
||||
if len(config.Keys) == 0 {
|
||||
return result, fmt.Errorf("secretbox provider has no valid keys")
|
||||
@ -603,7 +620,7 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
keyTransformers := []value.PrefixTransformer{}
|
||||
keyTransformers := []storagevalue.PrefixTransformer{}
|
||||
|
||||
for _, keyData := range config.Keys {
|
||||
keyData := keyData
|
||||
@ -621,18 +638,18 @@ func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration)
|
||||
|
||||
// Create a new PrefixTransformer for this key
|
||||
keyTransformers = append(keyTransformers,
|
||||
value.PrefixTransformer{
|
||||
storagevalue.PrefixTransformer{
|
||||
Transformer: secretbox.NewSecretboxTransformer(keyArray),
|
||||
Prefix: []byte(keyData.Name + ":"),
|
||||
})
|
||||
}
|
||||
|
||||
// Create a prefixTransformer which can choose between these keys
|
||||
keyTransformer := value.NewPrefixTransformers(
|
||||
keyTransformer := storagevalue.NewPrefixTransformers(
|
||||
fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
|
||||
|
||||
// Create a PrefixTransformer which shall later be put in a list with other providers
|
||||
result = value.PrefixTransformer{
|
||||
result = storagevalue.PrefixTransformer{
|
||||
Transformer: keyTransformer,
|
||||
Prefix: []byte(secretboxTransformerPrefixV1),
|
||||
}
|
||||
@ -665,13 +682,18 @@ func (s *kmsState) accumulate(other *kmsState) {
|
||||
// kmsPrefixTransformer creates a KMS transformer and probe based on the given KMS config.
|
||||
// It may launch multiple go routines whose lifecycle is controlled by ctx.
|
||||
// In case of an error, the caller is responsible for canceling ctx to clean up any go routines that may have been launched.
|
||||
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (value.PrefixTransformer, healthChecker, *kmsState, error) {
|
||||
func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfiguration) (storagevalue.PrefixTransformer, healthChecker, *kmsState, error) {
|
||||
kmsName := config.Name
|
||||
switch config.APIVersion {
|
||||
case kmsAPIVersionV1:
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv1) {
|
||||
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead. Set --feature-gates=KMSv1=true to use the deprecated KMSv1 feature.")
|
||||
}
|
||||
klog.InfoS("KMSv1 is deprecated and will only receive security updates going forward. Use KMSv2 instead.")
|
||||
|
||||
envelopeService, err := envelopeServiceFactory(ctx, config.Endpoint, config.Timeout.Duration)
|
||||
if err != nil {
|
||||
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
|
||||
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv1-Plugin's probe %q, error: %w", kmsName, err)
|
||||
}
|
||||
|
||||
probe := &kmsPluginProbe{
|
||||
@ -692,12 +714,12 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
|
||||
|
||||
case kmsAPIVersionV2:
|
||||
if !utilfeature.DefaultFeatureGate.Enabled(features.KMSv2) {
|
||||
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
|
||||
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2 plugin %q, KMSv2 feature is not enabled", kmsName)
|
||||
}
|
||||
|
||||
envelopeService, err := EnvelopeKMSv2ServiceFactory(ctx, config.Endpoint, config.Name, config.Timeout.Duration)
|
||||
if err != nil {
|
||||
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
|
||||
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMSv2-Plugin's probe %q, error: %w", kmsName, err)
|
||||
}
|
||||
|
||||
probe := &kmsv2PluginProbe{
|
||||
@ -710,45 +732,9 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
|
||||
// initialize state so that Load always works
|
||||
probe.state.Store(&envelopekmsv2.State{})
|
||||
|
||||
runProbeCheckAndLog := func(ctx context.Context) error {
|
||||
if err := probe.check(ctx); err != nil {
|
||||
klog.VDepth(1, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
primeAndProbeKMSv2(ctx, probe, kmsName)
|
||||
|
||||
// on the happy path where the plugin is healthy and available on server start,
|
||||
// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
|
||||
// ignore the error here since we want to support the plugin starting up async with the API server
|
||||
_ = runProbeCheckAndLog(ctx)
|
||||
// make sure that the plugin's key ID is reasonably up-to-date
|
||||
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
|
||||
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKMaxTTL
|
||||
go wait.PollUntilWithContext(
|
||||
ctx,
|
||||
kmsv2PluginHealthzPositiveInterval,
|
||||
func(ctx context.Context) (bool, error) {
|
||||
if err := runProbeCheckAndLog(ctx); err == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// TODO add integration test for quicker error poll on failure
|
||||
// if we fail, block the outer polling and start a new quicker poll inline
|
||||
// this limits the chance that our DEK expires during a transient failure
|
||||
_ = wait.PollUntilWithContext(
|
||||
ctx,
|
||||
kmsv2PluginHealthzNegativeInterval,
|
||||
func(ctx context.Context) (bool, error) {
|
||||
return runProbeCheckAndLog(ctx) == nil, nil
|
||||
},
|
||||
)
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
// using AES-GCM by default for encrypting data with KMSv2
|
||||
transformer := value.PrefixTransformer{
|
||||
transformer := storagevalue.PrefixTransformer{
|
||||
Transformer: envelopekmsv2.NewEnvelopeTransformer(envelopeService, kmsName, probe.getCurrentState),
|
||||
Prefix: []byte(kmsTransformerPrefixV2 + kmsName + ":"),
|
||||
}
|
||||
@ -759,12 +745,62 @@ func kmsPrefixTransformer(ctx context.Context, config *apiserverconfig.KMSConfig
|
||||
}, nil
|
||||
|
||||
default:
|
||||
return value.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
|
||||
return storagevalue.PrefixTransformer{}, nil, nil, fmt.Errorf("could not configure KMS plugin %q, unsupported KMS API version %q", kmsName, config.APIVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) value.PrefixTransformer {
|
||||
baseTransformerFunc := func(block cipher.Block) (value.Transformer, error) {
|
||||
func primeAndProbeKMSv2(ctx context.Context, probe *kmsv2PluginProbe, kmsName string) {
|
||||
runProbeCheckAndLog := func(ctx context.Context, depth int) error {
|
||||
if err := probe.check(ctx); err != nil {
|
||||
klog.VDepth(1+depth, 2).ErrorS(err, "kms plugin failed health check probe", "name", kmsName)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
blockAndProbeFastUntilSuccess := func(ctx context.Context) {
|
||||
_ = wait.PollUntilWithContext(
|
||||
ctx,
|
||||
kmsv2PluginHealthzNegativeInterval,
|
||||
func(ctx context.Context) (bool, error) {
|
||||
return runProbeCheckAndLog(ctx, 1) == nil, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
// on the happy path where the plugin is healthy and available on server start,
|
||||
// prime keyID and DEK by running the check inline once (this also prevents unit tests from flaking)
|
||||
errPrime := runProbeCheckAndLog(ctx, 0)
|
||||
|
||||
// if our initial attempt to prime failed, start trying to get to a valid state in the background ASAP
|
||||
// this prevents a slow start when the external healthz checker is configured to ignore the KMS healthz endpoint
|
||||
// since we want to support the plugin starting up async with the API server, this error is not fatal
|
||||
if errPrime != nil {
|
||||
go blockAndProbeFastUntilSuccess(ctx) // separate go routine to avoid blocking
|
||||
}
|
||||
|
||||
// make sure that the plugin's key ID is reasonably up-to-date
|
||||
// also, make sure that our DEK is up-to-date to with said key ID (if it expires the server will fail all writes)
|
||||
// if this background loop ever stops running, the server will become unfunctional after kmsv2PluginWriteDEKSourceMaxTTL
|
||||
go wait.PollUntilWithContext(
|
||||
ctx,
|
||||
kmsv2PluginHealthzPositiveInterval,
|
||||
func(ctx context.Context) (bool, error) {
|
||||
if err := runProbeCheckAndLog(ctx, 0); err == nil {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// TODO add integration test for quicker error poll on failure
|
||||
// if we fail, block the outer polling and start a new quicker poll inline
|
||||
// this limits the chance that our DEK expires during a transient failure
|
||||
blockAndProbeFastUntilSuccess(ctx)
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
|
||||
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) storagevalue.PrefixTransformer {
|
||||
baseTransformerFunc := func(block cipher.Block) (storagevalue.Transformer, error) {
|
||||
gcm, err := aestransformer.NewGCMTransformer(block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@ -777,15 +813,15 @@ func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelop
|
||||
return unionTransformers{gcm, aestransformer.NewCBCTransformer(block)}, nil
|
||||
}
|
||||
|
||||
return value.PrefixTransformer{
|
||||
return storagevalue.PrefixTransformer{
|
||||
Transformer: envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), baseTransformerFunc),
|
||||
Prefix: []byte(prefix + config.Name + ":"),
|
||||
}
|
||||
}
|
||||
|
||||
type unionTransformers []value.Transformer
|
||||
type unionTransformers []storagevalue.Transformer
|
||||
|
||||
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, stale bool, err error) {
|
||||
func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, stale bool, err error) {
|
||||
var errs []error
|
||||
for i := range u {
|
||||
transformer := u[i]
|
||||
@ -804,7 +840,7 @@ func (u unionTransformers) TransformFromStorage(ctx context.Context, data []byte
|
||||
return nil, false, fmt.Errorf("unionTransformers: unable to transform from storage")
|
||||
}
|
||||
|
||||
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) (out []byte, err error) {
|
||||
func (u unionTransformers) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) (out []byte, err error) {
|
||||
return u[0].TransformToStorage(ctx, data, dataCtx)
|
||||
}
|
||||
|
||||
@ -815,7 +851,7 @@ func computeEncryptionConfigHash(data []byte) string {
|
||||
return fmt.Sprintf("%x", sha256.Sum256(data))
|
||||
}
|
||||
|
||||
var _ ResourceTransformers = &DynamicTransformers{}
|
||||
var _ storagevalue.ResourceTransformers = &DynamicTransformers{}
|
||||
var _ healthz.HealthChecker = &DynamicTransformers{}
|
||||
|
||||
// DynamicTransformers holds transformers that may be dynamically updated via a single external actor, likely a controller.
|
||||
@ -825,7 +861,7 @@ type DynamicTransformers struct {
|
||||
}
|
||||
|
||||
type transformTracker struct {
|
||||
transformerOverrides map[schema.GroupResource]value.Transformer
|
||||
transformerOverrides map[schema.GroupResource]storagevalue.Transformer
|
||||
kmsPluginHealthzCheck healthz.HealthChecker
|
||||
closeTransformers context.CancelFunc
|
||||
kmsCloseGracePeriod time.Duration
|
||||
@ -833,7 +869,7 @@ type transformTracker struct {
|
||||
|
||||
// NewDynamicTransformers returns transformers, health checks for kms providers and an ability to close transformers.
|
||||
func NewDynamicTransformers(
|
||||
transformerOverrides map[schema.GroupResource]value.Transformer,
|
||||
transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
|
||||
kmsPluginHealthzCheck healthz.HealthChecker,
|
||||
closeTransformers context.CancelFunc,
|
||||
kmsCloseGracePeriod time.Duration,
|
||||
@ -864,7 +900,7 @@ func (d *DynamicTransformers) Name() string {
|
||||
}
|
||||
|
||||
// TransformerForResource returns the transformer for the given resource.
|
||||
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
|
||||
func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
|
||||
return &resourceTransformer{
|
||||
resource: resource,
|
||||
transformTracker: d.transformTracker,
|
||||
@ -873,7 +909,7 @@ func (d *DynamicTransformers) TransformerForResource(resource schema.GroupResour
|
||||
|
||||
// Set sets the transformer overrides. This method is not go routine safe and must only be called by the same, single caller throughout the lifetime of this object.
|
||||
func (d *DynamicTransformers) Set(
|
||||
transformerOverrides map[schema.GroupResource]value.Transformer,
|
||||
transformerOverrides map[schema.GroupResource]storagevalue.Transformer,
|
||||
closeTransformers context.CancelFunc,
|
||||
kmsPluginHealthzCheck healthz.HealthChecker,
|
||||
kmsCloseGracePeriod time.Duration,
|
||||
@ -898,34 +934,30 @@ func (d *DynamicTransformers) Set(
|
||||
}()
|
||||
}
|
||||
|
||||
var _ value.Transformer = &resourceTransformer{}
|
||||
var _ storagevalue.Transformer = &resourceTransformer{}
|
||||
|
||||
type resourceTransformer struct {
|
||||
resource schema.GroupResource
|
||||
transformTracker *atomic.Value
|
||||
}
|
||||
|
||||
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, bool, error) {
|
||||
func (r *resourceTransformer) TransformFromStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, bool, error) {
|
||||
return r.transformer().TransformFromStorage(ctx, data, dataCtx)
|
||||
}
|
||||
|
||||
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx value.Context) ([]byte, error) {
|
||||
func (r *resourceTransformer) TransformToStorage(ctx context.Context, data []byte, dataCtx storagevalue.Context) ([]byte, error) {
|
||||
return r.transformer().TransformToStorage(ctx, data, dataCtx)
|
||||
}
|
||||
|
||||
func (r *resourceTransformer) transformer() value.Transformer {
|
||||
func (r *resourceTransformer) transformer() storagevalue.Transformer {
|
||||
return transformerFromOverrides(r.transformTracker.Load().(*transformTracker).transformerOverrides, r.resource)
|
||||
}
|
||||
|
||||
type ResourceTransformers interface {
|
||||
TransformerForResource(resource schema.GroupResource) value.Transformer
|
||||
}
|
||||
var _ storagevalue.ResourceTransformers = &StaticTransformers{}
|
||||
|
||||
var _ ResourceTransformers = &StaticTransformers{}
|
||||
type StaticTransformers map[schema.GroupResource]storagevalue.Transformer
|
||||
|
||||
type StaticTransformers map[schema.GroupResource]value.Transformer
|
||||
|
||||
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) value.Transformer {
|
||||
func (s StaticTransformers) TransformerForResource(resource schema.GroupResource) storagevalue.Transformer {
|
||||
return transformerFromOverrides(s, resource)
|
||||
}
|
||||
|
||||
@ -934,7 +966,7 @@ var anyGroupAnyResource = schema.GroupResource{
|
||||
Resource: "*",
|
||||
}
|
||||
|
||||
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer {
|
||||
func transformerFromOverrides(transformerOverrides map[schema.GroupResource]storagevalue.Transformer, resource schema.GroupResource) storagevalue.Transformer {
|
||||
if transformer := transformerOverrides[resource]; transformer != nil {
|
||||
return transformer
|
||||
}
|
||||
|
||||
10
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go
generated
vendored
10
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/controller/controller.go
generated
vendored
@ -27,6 +27,7 @@ import (
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/apiserver/pkg/server/healthz"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig"
|
||||
"k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics"
|
||||
"k8s.io/client-go/util/workqueue"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
@ -163,16 +164,19 @@ func (d *DynamicKMSEncryptionConfigContent) processNextWorkItem(serverCtx contex
|
||||
ctx, closeTransformers := context.WithCancel(serverCtx)
|
||||
|
||||
defer func() {
|
||||
// TODO: increment success metric when updatedEffectiveConfig=true
|
||||
|
||||
// TODO can work queue metrics help here?
|
||||
|
||||
if !updatedEffectiveConfig {
|
||||
// avoid leaking if we're not using the newly constructed transformers (due to an error or them not being changed)
|
||||
closeTransformers()
|
||||
}
|
||||
|
||||
if updatedEffectiveConfig && err == nil {
|
||||
metrics.RecordEncryptionConfigAutomaticReloadSuccess()
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// TODO: increment failure metric
|
||||
metrics.RecordEncryptionConfigAutomaticReloadFailure()
|
||||
utilruntime.HandleError(fmt.Errorf("error processing encryption config file %s: %v", d.filePath, err))
|
||||
// add dummy item back to the queue to trigger file content processing.
|
||||
d.queue.AddRateLimited(key)
|
||||
|
||||
86
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go
generated
vendored
Normal file
86
vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/metrics/metrics.go
generated
vendored
Normal file
@ -0,0 +1,86 @@
|
||||
/*
|
||||
Copyright 2023 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"k8s.io/component-base/metrics"
|
||||
"k8s.io/component-base/metrics/legacyregistry"
|
||||
)
|
||||
|
||||
const (
|
||||
namespace = "apiserver"
|
||||
subsystem = "encryption_config_controller"
|
||||
)
|
||||
|
||||
var (
|
||||
encryptionConfigAutomaticReloadFailureTotal = metrics.NewCounter(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_failures_total",
|
||||
Help: "Total number of failed automatic reloads of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadSuccessTotal = metrics.NewCounter(
|
||||
&metrics.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_success_total",
|
||||
Help: "Total number of successful automatic reloads of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
)
|
||||
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds = metrics.NewGaugeVec(
|
||||
&metrics.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Subsystem: subsystem,
|
||||
Name: "automatic_reload_last_timestamp_seconds",
|
||||
Help: "Timestamp of the last successful or failed automatic reload of encryption configuration.",
|
||||
StabilityLevel: metrics.ALPHA,
|
||||
},
|
||||
[]string{"status"},
|
||||
)
|
||||
)
|
||||
|
||||
var registerMetrics sync.Once
|
||||
|
||||
func RegisterMetrics() {
|
||||
registerMetrics.Do(func() {
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadFailureTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadSuccessTotal)
|
||||
legacyregistry.MustRegister(encryptionConfigAutomaticReloadLastTimestampSeconds)
|
||||
})
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadFailure() {
|
||||
encryptionConfigAutomaticReloadFailureTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("failure")
|
||||
}
|
||||
|
||||
func RecordEncryptionConfigAutomaticReloadSuccess() {
|
||||
encryptionConfigAutomaticReloadSuccessTotal.Inc()
|
||||
recordEncryptionConfigAutomaticReloadTimestamp("success")
|
||||
}
|
||||
|
||||
func recordEncryptionConfigAutomaticReloadTimestamp(result string) {
|
||||
encryptionConfigAutomaticReloadLastTimestampSeconds.WithLabelValues(result).SetToCurrentTime()
|
||||
}
|
||||
Reference in New Issue
Block a user