rebase: update replaced k8s.io modules to v0.33.0

Signed-off-by: Niels de Vos <ndevos@ibm.com>
This commit is contained in:
Niels de Vos
2025-05-07 13:13:33 +02:00
committed by mergify[bot]
parent dd77e72800
commit 107407b44b
1723 changed files with 65035 additions and 175239 deletions

View File

@ -18,6 +18,7 @@ package api
import (
v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1beta1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
@ -29,12 +30,19 @@ type ResourceSlice struct {
}
type ResourceSliceSpec struct {
Driver UniqueString
Pool ResourcePool
NodeName UniqueString
NodeSelector *v1.NodeSelector
AllNodes bool
Devices []Device
Driver UniqueString
Pool ResourcePool
NodeName UniqueString
NodeSelector *v1.NodeSelector
AllNodes bool
Devices []Device
PerDeviceNodeSelection *bool
SharedCounters []CounterSet
}
type CounterSet struct {
Name UniqueString
Counters map[string]Counter
}
type ResourcePool struct {
@ -48,8 +56,18 @@ type Device struct {
}
type BasicDevice struct {
Attributes map[QualifiedName]DeviceAttribute
Capacity map[QualifiedName]DeviceCapacity
Attributes map[QualifiedName]DeviceAttribute
Capacity map[QualifiedName]DeviceCapacity
ConsumesCounters []DeviceCounterConsumption
NodeName *string
NodeSelector *v1.NodeSelector
AllNodes *bool
Taints []resourceapi.DeviceTaint
}
type DeviceCounterConsumption struct {
CounterSet UniqueString
Counters map[string]Counter
}
type QualifiedName string
@ -66,3 +84,22 @@ type DeviceAttribute struct {
type DeviceCapacity struct {
Value resource.Quantity
}
type Counter struct {
Value resource.Quantity
}
type DeviceTaint struct {
Key string
Value string
Effect DeviceTaintEffect
TimeAdded *metav1.Time
}
type DeviceTaintEffect string
const (
DeviceTaintEffectNoSchedule DeviceTaintEffect = "NoSchedule"
DeviceTaintEffectNoExecute DeviceTaintEffect = "NoExecute"
)

View File

@ -26,6 +26,7 @@ import (
v1 "k8s.io/api/core/v1"
v1beta1 "k8s.io/api/resource/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
conversion "k8s.io/apimachinery/pkg/conversion"
runtime "k8s.io/apimachinery/pkg/runtime"
)
@ -47,6 +48,26 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*Counter)(nil), (*v1beta1.Counter)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_Counter_To_v1beta1_Counter(a.(*Counter), b.(*v1beta1.Counter), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.Counter)(nil), (*Counter)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_Counter_To_api_Counter(a.(*v1beta1.Counter), b.(*Counter), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*CounterSet)(nil), (*v1beta1.CounterSet)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_CounterSet_To_v1beta1_CounterSet(a.(*CounterSet), b.(*v1beta1.CounterSet), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.CounterSet)(nil), (*CounterSet)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_CounterSet_To_api_CounterSet(a.(*v1beta1.CounterSet), b.(*CounterSet), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*Device)(nil), (*v1beta1.Device)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_Device_To_v1beta1_Device(a.(*Device), b.(*v1beta1.Device), scope)
}); err != nil {
@ -77,6 +98,26 @@ func RegisterConversions(s *runtime.Scheme) error {
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*DeviceCounterConsumption)(nil), (*v1beta1.DeviceCounterConsumption)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption(a.(*DeviceCounterConsumption), b.(*v1beta1.DeviceCounterConsumption), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.DeviceCounterConsumption)(nil), (*DeviceCounterConsumption)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption(a.(*v1beta1.DeviceCounterConsumption), b.(*DeviceCounterConsumption), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*DeviceTaint)(nil), (*v1beta1.DeviceTaint)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_DeviceTaint_To_v1beta1_DeviceTaint(a.(*DeviceTaint), b.(*v1beta1.DeviceTaint), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*v1beta1.DeviceTaint)(nil), (*DeviceTaint)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_v1beta1_DeviceTaint_To_api_DeviceTaint(a.(*v1beta1.DeviceTaint), b.(*DeviceTaint), scope)
}); err != nil {
return err
}
if err := s.AddGeneratedConversionFunc((*ResourcePool)(nil), (*v1beta1.ResourcePool)(nil), func(a, b interface{}, scope conversion.Scope) error {
return Convert_api_ResourcePool_To_v1beta1_ResourcePool(a.(*ResourcePool), b.(*v1beta1.ResourcePool), scope)
}); err != nil {
@ -123,6 +164,21 @@ func RegisterConversions(s *runtime.Scheme) error {
func autoConvert_api_BasicDevice_To_v1beta1_BasicDevice(in *BasicDevice, out *v1beta1.BasicDevice, s conversion.Scope) error {
out.Attributes = *(*map[v1beta1.QualifiedName]v1beta1.DeviceAttribute)(unsafe.Pointer(&in.Attributes))
out.Capacity = *(*map[v1beta1.QualifiedName]v1beta1.DeviceCapacity)(unsafe.Pointer(&in.Capacity))
if in.ConsumesCounters != nil {
in, out := &in.ConsumesCounters, &out.ConsumesCounters
*out = make([]v1beta1.DeviceCounterConsumption, len(*in))
for i := range *in {
if err := Convert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ConsumesCounters = nil
}
out.NodeName = (*string)(unsafe.Pointer(in.NodeName))
out.NodeSelector = (*v1.NodeSelector)(unsafe.Pointer(in.NodeSelector))
out.AllNodes = (*bool)(unsafe.Pointer(in.AllNodes))
out.Taints = *(*[]v1beta1.DeviceTaint)(unsafe.Pointer(&in.Taints))
return nil
}
@ -134,6 +190,21 @@ func Convert_api_BasicDevice_To_v1beta1_BasicDevice(in *BasicDevice, out *v1beta
func autoConvert_v1beta1_BasicDevice_To_api_BasicDevice(in *v1beta1.BasicDevice, out *BasicDevice, s conversion.Scope) error {
out.Attributes = *(*map[QualifiedName]DeviceAttribute)(unsafe.Pointer(&in.Attributes))
out.Capacity = *(*map[QualifiedName]DeviceCapacity)(unsafe.Pointer(&in.Capacity))
if in.ConsumesCounters != nil {
in, out := &in.ConsumesCounters, &out.ConsumesCounters
*out = make([]DeviceCounterConsumption, len(*in))
for i := range *in {
if err := Convert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.ConsumesCounters = nil
}
out.NodeName = (*string)(unsafe.Pointer(in.NodeName))
out.NodeSelector = (*v1.NodeSelector)(unsafe.Pointer(in.NodeSelector))
out.AllNodes = (*bool)(unsafe.Pointer(in.AllNodes))
out.Taints = *(*[]v1beta1.DeviceTaint)(unsafe.Pointer(&in.Taints))
return nil
}
@ -142,11 +213,65 @@ func Convert_v1beta1_BasicDevice_To_api_BasicDevice(in *v1beta1.BasicDevice, out
return autoConvert_v1beta1_BasicDevice_To_api_BasicDevice(in, out, s)
}
func autoConvert_api_Counter_To_v1beta1_Counter(in *Counter, out *v1beta1.Counter, s conversion.Scope) error {
out.Value = in.Value
return nil
}
// Convert_api_Counter_To_v1beta1_Counter is an autogenerated conversion function.
func Convert_api_Counter_To_v1beta1_Counter(in *Counter, out *v1beta1.Counter, s conversion.Scope) error {
return autoConvert_api_Counter_To_v1beta1_Counter(in, out, s)
}
func autoConvert_v1beta1_Counter_To_api_Counter(in *v1beta1.Counter, out *Counter, s conversion.Scope) error {
out.Value = in.Value
return nil
}
// Convert_v1beta1_Counter_To_api_Counter is an autogenerated conversion function.
func Convert_v1beta1_Counter_To_api_Counter(in *v1beta1.Counter, out *Counter, s conversion.Scope) error {
return autoConvert_v1beta1_Counter_To_api_Counter(in, out, s)
}
func autoConvert_api_CounterSet_To_v1beta1_CounterSet(in *CounterSet, out *v1beta1.CounterSet, s conversion.Scope) error {
if err := Convert_api_UniqueString_To_string(&in.Name, &out.Name, s); err != nil {
return err
}
out.Counters = *(*map[string]v1beta1.Counter)(unsafe.Pointer(&in.Counters))
return nil
}
// Convert_api_CounterSet_To_v1beta1_CounterSet is an autogenerated conversion function.
func Convert_api_CounterSet_To_v1beta1_CounterSet(in *CounterSet, out *v1beta1.CounterSet, s conversion.Scope) error {
return autoConvert_api_CounterSet_To_v1beta1_CounterSet(in, out, s)
}
func autoConvert_v1beta1_CounterSet_To_api_CounterSet(in *v1beta1.CounterSet, out *CounterSet, s conversion.Scope) error {
if err := Convert_string_To_api_UniqueString(&in.Name, &out.Name, s); err != nil {
return err
}
out.Counters = *(*map[string]Counter)(unsafe.Pointer(&in.Counters))
return nil
}
// Convert_v1beta1_CounterSet_To_api_CounterSet is an autogenerated conversion function.
func Convert_v1beta1_CounterSet_To_api_CounterSet(in *v1beta1.CounterSet, out *CounterSet, s conversion.Scope) error {
return autoConvert_v1beta1_CounterSet_To_api_CounterSet(in, out, s)
}
func autoConvert_api_Device_To_v1beta1_Device(in *Device, out *v1beta1.Device, s conversion.Scope) error {
if err := Convert_api_UniqueString_To_string(&in.Name, &out.Name, s); err != nil {
return err
}
out.Basic = (*v1beta1.BasicDevice)(unsafe.Pointer(in.Basic))
if in.Basic != nil {
in, out := &in.Basic, &out.Basic
*out = new(v1beta1.BasicDevice)
if err := Convert_api_BasicDevice_To_v1beta1_BasicDevice(*in, *out, s); err != nil {
return err
}
} else {
out.Basic = nil
}
return nil
}
@ -159,7 +284,15 @@ func autoConvert_v1beta1_Device_To_api_Device(in *v1beta1.Device, out *Device, s
if err := Convert_string_To_api_UniqueString(&in.Name, &out.Name, s); err != nil {
return err
}
out.Basic = (*BasicDevice)(unsafe.Pointer(in.Basic))
if in.Basic != nil {
in, out := &in.Basic, &out.Basic
*out = new(BasicDevice)
if err := Convert_v1beta1_BasicDevice_To_api_BasicDevice(*in, *out, s); err != nil {
return err
}
} else {
out.Basic = nil
}
return nil
}
@ -214,6 +347,58 @@ func Convert_v1beta1_DeviceCapacity_To_api_DeviceCapacity(in *v1beta1.DeviceCapa
return autoConvert_v1beta1_DeviceCapacity_To_api_DeviceCapacity(in, out, s)
}
func autoConvert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption(in *DeviceCounterConsumption, out *v1beta1.DeviceCounterConsumption, s conversion.Scope) error {
if err := Convert_api_UniqueString_To_string(&in.CounterSet, &out.CounterSet, s); err != nil {
return err
}
out.Counters = *(*map[string]v1beta1.Counter)(unsafe.Pointer(&in.Counters))
return nil
}
// Convert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption is an autogenerated conversion function.
func Convert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption(in *DeviceCounterConsumption, out *v1beta1.DeviceCounterConsumption, s conversion.Scope) error {
return autoConvert_api_DeviceCounterConsumption_To_v1beta1_DeviceCounterConsumption(in, out, s)
}
func autoConvert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption(in *v1beta1.DeviceCounterConsumption, out *DeviceCounterConsumption, s conversion.Scope) error {
if err := Convert_string_To_api_UniqueString(&in.CounterSet, &out.CounterSet, s); err != nil {
return err
}
out.Counters = *(*map[string]Counter)(unsafe.Pointer(&in.Counters))
return nil
}
// Convert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption is an autogenerated conversion function.
func Convert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption(in *v1beta1.DeviceCounterConsumption, out *DeviceCounterConsumption, s conversion.Scope) error {
return autoConvert_v1beta1_DeviceCounterConsumption_To_api_DeviceCounterConsumption(in, out, s)
}
func autoConvert_api_DeviceTaint_To_v1beta1_DeviceTaint(in *DeviceTaint, out *v1beta1.DeviceTaint, s conversion.Scope) error {
out.Key = in.Key
out.Value = in.Value
out.Effect = v1beta1.DeviceTaintEffect(in.Effect)
out.TimeAdded = (*metav1.Time)(unsafe.Pointer(in.TimeAdded))
return nil
}
// Convert_api_DeviceTaint_To_v1beta1_DeviceTaint is an autogenerated conversion function.
func Convert_api_DeviceTaint_To_v1beta1_DeviceTaint(in *DeviceTaint, out *v1beta1.DeviceTaint, s conversion.Scope) error {
return autoConvert_api_DeviceTaint_To_v1beta1_DeviceTaint(in, out, s)
}
func autoConvert_v1beta1_DeviceTaint_To_api_DeviceTaint(in *v1beta1.DeviceTaint, out *DeviceTaint, s conversion.Scope) error {
out.Key = in.Key
out.Value = in.Value
out.Effect = DeviceTaintEffect(in.Effect)
out.TimeAdded = (*metav1.Time)(unsafe.Pointer(in.TimeAdded))
return nil
}
// Convert_v1beta1_DeviceTaint_To_api_DeviceTaint is an autogenerated conversion function.
func Convert_v1beta1_DeviceTaint_To_api_DeviceTaint(in *v1beta1.DeviceTaint, out *DeviceTaint, s conversion.Scope) error {
return autoConvert_v1beta1_DeviceTaint_To_api_DeviceTaint(in, out, s)
}
func autoConvert_api_ResourcePool_To_v1beta1_ResourcePool(in *ResourcePool, out *v1beta1.ResourcePool, s conversion.Scope) error {
if err := Convert_api_UniqueString_To_string(&in.Name, &out.Name, s); err != nil {
return err
@ -291,6 +476,18 @@ func autoConvert_api_ResourceSliceSpec_To_v1beta1_ResourceSliceSpec(in *Resource
} else {
out.Devices = nil
}
out.PerDeviceNodeSelection = (*bool)(unsafe.Pointer(in.PerDeviceNodeSelection))
if in.SharedCounters != nil {
in, out := &in.SharedCounters, &out.SharedCounters
*out = make([]v1beta1.CounterSet, len(*in))
for i := range *in {
if err := Convert_api_CounterSet_To_v1beta1_CounterSet(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.SharedCounters = nil
}
return nil
}
@ -322,6 +519,18 @@ func autoConvert_v1beta1_ResourceSliceSpec_To_api_ResourceSliceSpec(in *v1beta1.
} else {
out.Devices = nil
}
out.PerDeviceNodeSelection = (*bool)(unsafe.Pointer(in.PerDeviceNodeSelection))
if in.SharedCounters != nil {
in, out := &in.SharedCounters, &out.SharedCounters
*out = make([]CounterSet, len(*in))
for i := range *in {
if err := Convert_v1beta1_CounterSet_To_api_CounterSet(&(*in)[i], &(*out)[i], s); err != nil {
return err
}
}
} else {
out.SharedCounters = nil
}
return nil
}

View File

@ -43,6 +43,8 @@ func NewCache(maxCacheEntries int) *Cache {
// GetOrCompile checks whether the cache already has a compilation result
// and returns that if available. Otherwise it compiles, stores successful
// results and returns the new result.
//
// Cost estimation is disabled.
func (c *Cache) GetOrCompile(expression string) CompilationResult {
// Compiling a CEL expression is expensive enough that it is cheaper
// to lock a mutex than doing it several times in parallel.
@ -55,7 +57,7 @@ func (c *Cache) GetOrCompile(expression string) CompilationResult {
return *cached
}
expr := GetCompiler().CompileCELExpression(expression, Options{})
expr := GetCompiler().CompileCELExpression(expression, Options{DisableCostEstimation: true})
if expr.Error == nil {
c.add(expression, &expr)
}

View File

@ -20,24 +20,27 @@ import (
"context"
"errors"
"fmt"
"math"
"reflect"
"strings"
"sync"
"github.com/blang/semver/v4"
"github.com/google/cel-go/cel"
"github.com/google/cel-go/checker"
"github.com/google/cel-go/common/types"
"github.com/google/cel-go/common/types/ref"
"github.com/google/cel-go/common/types/traits"
"github.com/google/cel-go/ext"
"k8s.io/utils/ptr"
resourceapi "k8s.io/api/resource/v1beta1"
"k8s.io/apimachinery/pkg/util/version"
celconfig "k8s.io/apiserver/pkg/apis/cel"
apiservercel "k8s.io/apiserver/pkg/cel"
"k8s.io/apiserver/pkg/cel/environment"
"k8s.io/apiserver/pkg/cel/library"
"k8s.io/utils/ptr"
)
const (
@ -50,6 +53,23 @@ const (
var (
lazyCompilerInit sync.Once
lazyCompiler *compiler
// A variant of AnyType = https://github.com/kubernetes/kubernetes/blob/ec2e0de35a298363872897e5904501b029817af3/staging/src/k8s.io/apiserver/pkg/cel/types.go#L550:
// unknown actual type (could be bool, int, string, etc.) but with a known maximum size.
attributeType = withMaxElements(apiservercel.AnyType, resourceapi.DeviceAttributeMaxValueLength)
// Other strings also have a known maximum size.
domainType = withMaxElements(apiservercel.StringType, resourceapi.DeviceMaxDomainLength)
idType = withMaxElements(apiservercel.StringType, resourceapi.DeviceMaxIDLength)
driverType = withMaxElements(apiservercel.StringType, resourceapi.DriverNameMaxLength)
// Each map is bound by the maximum number of different attributes.
innerAttributesMapType = apiservercel.NewMapType(idType, attributeType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice)
outerAttributesMapType = apiservercel.NewMapType(domainType, innerAttributesMapType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice)
// Same for capacity.
innerCapacityMapType = apiservercel.NewMapType(idType, apiservercel.QuantityDeclType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice)
outerCapacityMapType = apiservercel.NewMapType(domainType, innerCapacityMapType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice)
)
func GetCompiler() *compiler {
@ -85,11 +105,12 @@ type Device struct {
}
type compiler struct {
envset *environment.EnvSet
}
func newCompiler() *compiler {
return &compiler{envset: mustBuildEnv()}
// deviceType is a definition for the type of the `device` variable.
// This is needed for the cost estimator. Both are currently version-independent.
// If that ever changes, some additional logic might be needed to make
// cost estimates version-dependent.
deviceType *apiservercel.DeclType
envset *environment.EnvSet
}
// Options contains several additional parameters
@ -101,6 +122,10 @@ type Options struct {
// CostLimit allows overriding the default runtime cost limit [resourceapi.CELSelectorExpressionMaxCost].
CostLimit *uint64
// DisableCostEstimation can be set to skip estimating the worst-case CEL cost.
// If disabled or after an error, [CompilationResult.MaxCost] will be set to [math.Uint64].
DisableCostEstimation bool
}
// CompileCELExpression returns a compiled CEL expression. It evaluates to bool.
@ -114,6 +139,7 @@ func (c compiler) CompileCELExpression(expression string, options Options) Compi
Detail: errorString,
},
Expression: expression,
MaxCost: math.MaxUint64,
}
}
@ -122,10 +148,6 @@ func (c compiler) CompileCELExpression(expression string, options Options) Compi
return resultError(fmt.Sprintf("unexpected error loading CEL environment: %v", err), apiservercel.ErrorTypeInternal)
}
// We don't have a SizeEstimator. The potential size of the input (= a
// device) is already declared in the definition of the environment.
estimator := &library.CostEstimator{}
ast, issues := env.Compile(expression)
if issues != nil {
return resultError("compilation failed: "+issues.String(), apiservercel.ErrorTypeInvalid)
@ -157,18 +179,28 @@ func (c compiler) CompileCELExpression(expression string, options Options) Compi
OutputType: ast.OutputType(),
Environment: env,
emptyMapVal: env.CELTypeAdapter().NativeToValue(map[string]any{}),
MaxCost: math.MaxUint64,
}
costEst, err := env.EstimateCost(ast, estimator)
if err != nil {
compilationResult.Error = &apiservercel.Error{Type: apiservercel.ErrorTypeInternal, Detail: "cost estimation failed: " + err.Error()}
return compilationResult
if !options.DisableCostEstimation {
// We don't have a SizeEstimator. The potential size of the input (= a
// device) is already declared in the definition of the environment.
estimator := c.newCostEstimator()
costEst, err := env.EstimateCost(ast, estimator)
if err != nil {
compilationResult.Error = &apiservercel.Error{Type: apiservercel.ErrorTypeInternal, Detail: "cost estimation failed: " + err.Error()}
return compilationResult
}
compilationResult.MaxCost = costEst.Max
}
compilationResult.MaxCost = costEst.Max
return compilationResult
}
func (c *compiler) newCostEstimator() *library.CostEstimator {
return &library.CostEstimator{SizeEstimator: &sizeEstimator{compiler: c}}
}
// getAttributeValue returns the native representation of the one value that
// should be stored in the attribute, otherwise an error. An error is
// also returned when there is no supported value.
@ -241,7 +273,7 @@ func (c CompilationResult) DeviceMatches(ctx context.Context, input Device) (boo
return resultBool, details, nil
}
func mustBuildEnv() *environment.EnvSet {
func newCompiler() *compiler {
envset := environment.MustBaseEnvSet(environment.DefaultCompatibilityVersion(), true /* strictCost */)
field := func(name string, declType *apiservercel.DeclType, required bool) *apiservercel.DeclField {
return apiservercel.NewDeclField(name, declType, required, nil, nil)
@ -253,10 +285,11 @@ func mustBuildEnv() *environment.EnvSet {
}
return result
}
deviceType := apiservercel.NewObjectType("kubernetes.DRADevice", fields(
field(driverVar, apiservercel.StringType, true),
field(attributesVar, apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewMapType(apiservercel.StringType, apiservercel.AnyType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice), resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice), true),
field(capacityVar, apiservercel.NewMapType(apiservercel.StringType, apiservercel.NewMapType(apiservercel.StringType, apiservercel.QuantityDeclType, resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice), resourceapi.ResourceSliceMaxAttributesAndCapacitiesPerDevice), true),
field(driverVar, driverType, true),
field(attributesVar, outerAttributesMapType, true),
field(capacityVar, outerCapacityMapType, true),
))
versioned := []environment.VersionedOptions{
@ -265,8 +298,6 @@ func mustBuildEnv() *environment.EnvSet {
EnvOptions: []cel.EnvOption{
cel.Variable(deviceVar, deviceType.CelType()),
environment.UnversionedLib(library.SemverLib),
// https://pkg.go.dev/github.com/google/cel-go/ext#Bindings
//
// This is useful to simplify attribute lookups because the
@ -279,12 +310,34 @@ func mustBuildEnv() *environment.EnvSet {
deviceType,
},
},
{
IntroducedVersion: version.MajorMinor(1, 31),
// This library has added to base environment of Kubernetes
// in 1.33 at version 1. It will continue to be available for
// use in this environment, but does not need to be included
// directly since it becomes available indirectly via the base
// environment shared across Kubernetes.
// In Kubernetes 1.34, version 1 feature of this library will
// become available, and will be rollback safe to 1.33.
// TODO: In Kubernetes 1.34: Add compile tests that demonstrate that
// `isSemver("v1.0.0", true)` and `semver("v1.0.0", true)` are supported.
RemovedVersion: version.MajorMinor(1, 33),
EnvOptions: []cel.EnvOption{
library.SemverLib(library.SemverVersion(0)),
},
},
}
envset, err := envset.Extend(versioned...)
if err != nil {
panic(fmt.Errorf("internal error building CEL environment: %w", err))
}
return envset
return &compiler{envset: envset, deviceType: deviceType}
}
func withMaxElements(in *apiservercel.DeclType, maxElements uint64) *apiservercel.DeclType {
out := *in
out.MaxElements = int64(maxElements)
return &out
}
// parseQualifiedName splits into domain and identified, using the default domain
@ -322,3 +375,67 @@ func (m mapper) Find(key ref.Val) (ref.Val, bool) {
return m.defaultValue, true
}
// sizeEstimator tells the cost estimator the maximum size of maps or strings accessible through the `device` variable.
// Without this, the maximum string size of e.g. `device.attributes["dra.example.com"].services` would be unknown.
//
// sizeEstimator is derived from the sizeEstimator in k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel.
type sizeEstimator struct {
compiler *compiler
}
func (s *sizeEstimator) EstimateSize(element checker.AstNode) *checker.SizeEstimate {
path := element.Path()
if len(path) == 0 {
// Path() can return an empty list, early exit if it does since we can't
// provide size estimates when that happens
return nil
}
// The estimator provides information about the environment's variable(s).
var currentNode *apiservercel.DeclType
switch path[0] {
case deviceVar:
currentNode = s.compiler.deviceType
default:
// Unknown root, shouldn't happen.
return nil
}
// Cut off initial variable from path, it was checked above.
for _, name := range path[1:] {
switch name {
case "@items", "@values":
if currentNode.ElemType == nil {
return nil
}
currentNode = currentNode.ElemType
case "@keys":
if currentNode.KeyType == nil {
return nil
}
currentNode = currentNode.KeyType
default:
field, ok := currentNode.Fields[name]
if !ok {
// If this is an attribute map, then we know that all elements
// have the same maximum size as set in attributeType, regardless
// of their name.
if currentNode.ElemType == attributeType {
currentNode = attributeType
continue
}
return nil
}
if field.Type == nil {
return nil
}
currentNode = field.Type
}
}
return &checker.SizeEstimate{Min: 0, Max: uint64(currentNode.MaxElements)}
}
func (s *sizeEstimator) EstimateCallCost(function, overloadID string, target *checker.AstNode, args []checker.AstNode) *checker.CallEstimate {
return nil
}

View File

@ -0,0 +1,112 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// TODO: This is duplicated from ./pkg/scheduler/util/queue because that
// package is not allowed to be imported here.
package queue
const (
// normalSize limits the size of the buffer that is kept
// for reuse.
normalSize = 4
)
// FIFO implements a first-in-first-out queue with unbounded size.
// The null FIFO is a valid empty queue.
//
// Access must be protected by the caller when used concurrently by
// different goroutines, the queue itself implements no locking.
type FIFO[T any] struct {
// elements contains a buffer for elements which have been
// pushed and not popped yet. Two scenarios are possible:
// - one chunk in the middle (start <= end)
// - one chunk at the end, followed by one chunk at the
// beginning (end <= start)
//
// start == end can be either an empty queue or a completely
// full one (with two chunks).
elements []T
// len counts the number of elements which have been pushed and
// not popped yet.
len int
// start is the index of the first valid element.
start int
// end is the index after the last valid element.
end int
}
func (q *FIFO[T]) Len() int {
return q.len
}
func (q *FIFO[T]) Push(element T) {
size := len(q.elements)
if q.len == size {
// Need larger buffer.
newSize := size * 2
if newSize == 0 {
newSize = normalSize
}
elements := make([]T, newSize)
if q.start == 0 {
copy(elements, q.elements)
} else {
copy(elements, q.elements[q.start:])
copy(elements[len(q.elements)-q.start:], q.elements[0:q.end])
}
q.start = 0
q.end = q.len
q.elements = elements
size = newSize
}
if q.end == size {
// Wrap around.
q.elements[0] = element
q.end = 1
q.len++
return
}
q.elements[q.end] = element
q.end++
q.len++
}
func (q *FIFO[T]) Pop() (element T, ok bool) {
if q.len == 0 {
return
}
element = q.elements[q.start]
q.start++
if q.start == len(q.elements) {
// Wrap around.
q.start = 0
}
q.len--
// Once it is empty, shrink down to avoid hanging onto
// a large buffer forever.
if q.len == 0 && len(q.elements) > normalSize {
q.elements = make([]T, normalSize)
q.start = 0
q.end = 0
}
ok = true
return
}

View File

@ -0,0 +1,53 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package resourceclaim
import (
resourceapi "k8s.io/api/resource/v1beta1"
)
// ToleratesTaint checks if the toleration tolerates the taint.
// The matching follows the rules below:
//
// 1. Empty toleration.effect means to match all taint effects,
// otherwise taint effect must equal to toleration.effect.
// 2. If toleration.operator is 'Exists', it means to match all taint values.
// 3. Empty toleration.key means to match all taint keys.
// If toleration.key is empty, toleration.operator must be 'Exists';
// this combination means to match all taint values and all taint keys.
func ToleratesTaint(toleration resourceapi.DeviceToleration, taint resourceapi.DeviceTaint) bool {
// This code was copied from https://github.com/kubernetes/kubernetes/blob/f007012f5fe49e40ae0596cf463a8e7b247b3357/staging/src/k8s.io/api/core/v1/toleration.go#L39-L56.
// It wasn't placed in the resourceapi package because code related to logic
// doesn't belong there.
if toleration.Effect != "" && toleration.Effect != taint.Effect {
return false
}
if toleration.Key != "" && toleration.Key != taint.Key {
return false
}
switch toleration.Operator {
// Empty operator means Equal, should be set because of defaulting.
case "", resourceapi.DeviceTolerationOpEqual:
return toleration.Value == taint.Value
case resourceapi.DeviceTolerationOpExists:
return true
default:
return false
}
}

View File

@ -26,6 +26,8 @@ package resourceclaim
import (
"errors"
"fmt"
"slices"
"strings"
v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1beta1"
@ -106,3 +108,36 @@ func CanBeReserved(claim *resourceapi.ResourceClaim) bool {
// Currently no restrictions on sharing...
return true
}
// BaseRequestRef returns the request name if the reference is to a top-level
// request and the name of the parent request if the reference is to a subrequest.
func BaseRequestRef(requestRef string) string {
segments := strings.Split(requestRef, "/")
return segments[0]
}
// ConfigForResult returns the configs that are applicable to device
// allocated for the provided result.
func ConfigForResult(deviceConfigurations []resourceapi.DeviceAllocationConfiguration, result resourceapi.DeviceRequestAllocationResult) []resourceapi.DeviceAllocationConfiguration {
var configs []resourceapi.DeviceAllocationConfiguration
for _, deviceConfiguration := range deviceConfigurations {
if deviceConfiguration.Opaque != nil &&
isMatch(deviceConfiguration.Requests, result.Request) {
configs = append(configs, deviceConfiguration)
}
}
return configs
}
func isMatch(requests []string, requestRef string) bool {
if len(requests) == 0 {
return true
}
if slices.Contains(requests, requestRef) {
return true
}
baseRequestRef := BaseRequestRef(requestRef)
return slices.Contains(requests, baseRequestRef)
}

View File

@ -0,0 +1,798 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tracker
import (
"context"
"errors"
"fmt"
"slices"
"sync"
"github.com/google/go-cmp/cmp" //nolint:depguard
v1 "k8s.io/api/core/v1"
resourcealphaapi "k8s.io/api/resource/v1alpha3"
resourceapi "k8s.io/api/resource/v1beta1"
labels "k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
resourcealphainformers "k8s.io/client-go/informers/resource/v1alpha3"
resourceinformers "k8s.io/client-go/informers/resource/v1beta1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
resourcelisters "k8s.io/client-go/listers/resource/v1beta1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/dynamic-resource-allocation/cel"
"k8s.io/dynamic-resource-allocation/internal/queue"
"k8s.io/klog/v2"
"k8s.io/utils/ptr"
)
const (
driverPoolDeviceIndexName = "driverPoolDevice"
anyDriver = "*"
anyPool = "*"
anyDevice = "*"
)
// Tracker maintains a view of ResourceSlice objects with matching
// DeviceTaintRules applied. It is backed by informers to process
// potential changes to resolved ResourceSlices asynchronously.
type Tracker struct {
enableDeviceTaints bool
resourceSliceLister resourcelisters.ResourceSliceLister
resourceSlices cache.SharedIndexInformer
resourceSlicesHandle cache.ResourceEventHandlerRegistration
deviceTaints cache.SharedIndexInformer
deviceTaintsHandle cache.ResourceEventHandlerRegistration
deviceClasses cache.SharedIndexInformer
deviceClassesHandle cache.ResourceEventHandlerRegistration
celCache *cel.Cache
patchedResourceSlices cache.Store
broadcaster record.EventBroadcaster
recorder record.EventRecorder
// handleError usually refers to [utilruntime.HandleErrorWithContext] but
// may be overridden in tests.
handleError func(context.Context, error, string, ...any)
// Synchronizes updates to these fields related to event handlers.
rwMutex sync.RWMutex
// All registered event handlers.
eventHandlers []cache.ResourceEventHandler
// The eventQueue contains functions which deliver an event to one
// event handler.
//
// These functions must be invoked while *not locking* rwMutex because
// the event handlers are allowed to access the cache. Holding rwMutex
// then would cause a deadlock.
//
// New functions get added as part of processing a cache update while
// the rwMutex is locked. Each function which adds something to the queue
// also drains the queue before returning, therefore it is guaranteed
// that all event handlers get notified immediately (useful for unit
// testing).
//
// A channel cannot be used here because it cannot have an unbounded
// capacity. This could lead to a deadlock (writer holds rwMutex,
// gets blocked because capacity is exhausted, reader is in a handler
// which tries to lock the rwMutex). Writing into such a channel
// while not holding the rwMutex doesn't work because in-order delivery
// of events would no longer be guaranteed.
eventQueue queue.FIFO[func()]
}
// Options configure a [Tracker].
type Options struct {
// EnableDeviceTaints controls whether DeviceTaintRules
// will be reflected in ResourceSlices reported by the tracker.
//
// If false, then TaintInformer and ClassInformer
// are not needed. The tracker turns into
// a thin wrapper around the underlying
// SliceInformer, with no processing of its own.
EnableDeviceTaints bool
SliceInformer resourceinformers.ResourceSliceInformer
TaintInformer resourcealphainformers.DeviceTaintRuleInformer
ClassInformer resourceinformers.DeviceClassInformer
// KubeClient is used to generate Events when CEL expressions
// encounter runtime errors.
KubeClient kubernetes.Interface
}
// StartTracker creates and initializes informers for a new [Tracker].
func StartTracker(ctx context.Context, opts Options) (finalT *Tracker, finalErr error) {
if !opts.EnableDeviceTaints {
// Minimal wrapper. All public methods shortcut by calling the underlying informer.
return &Tracker{
resourceSliceLister: opts.SliceInformer.Lister(),
resourceSlices: opts.SliceInformer.Informer(),
}, nil
}
t, err := newTracker(ctx, opts)
if err != nil {
return nil, err
}
defer func() {
// If we don't return the tracker, stop the partially initialized instance.
if finalErr != nil {
t.Stop()
}
}()
if err := t.initInformers(ctx); err != nil {
return nil, fmt.Errorf("initialize informers: %w", err)
}
return t, nil
}
// newTracker is used in testing to construct a tracker without informer event handlers.
func newTracker(ctx context.Context, opts Options) (finalT *Tracker, finalErr error) {
t := &Tracker{
enableDeviceTaints: opts.EnableDeviceTaints,
resourceSliceLister: opts.SliceInformer.Lister(),
resourceSlices: opts.SliceInformer.Informer(),
deviceTaints: opts.TaintInformer.Informer(),
deviceClasses: opts.ClassInformer.Informer(),
celCache: cel.NewCache(10),
patchedResourceSlices: cache.NewStore(cache.MetaNamespaceKeyFunc),
handleError: utilruntime.HandleErrorWithContext,
}
defer func() {
// If we don't return the tracker, stop the partially initialized instance.
if finalErr != nil {
t.Stop()
}
}()
err := t.resourceSlices.AddIndexers(cache.Indexers{driverPoolDeviceIndexName: sliceDriverPoolDeviceIndexFunc})
if err != nil {
return nil, fmt.Errorf("failed to add %s index to ResourceSlice informer: %w", driverPoolDeviceIndexName, err)
}
// KubeClient is not always set in unit tests.
if opts.KubeClient != nil {
t.broadcaster = record.NewBroadcaster(record.WithContext(ctx))
t.broadcaster.StartLogging(klog.Infof)
t.broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: opts.KubeClient.CoreV1().Events("")})
t.recorder = t.broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "resource_slice_tracker"})
}
return t, nil
}
// initInformers adds event handlers to a tracker constructed with newTracker.
func (t *Tracker) initInformers(ctx context.Context) error {
var err error
sliceHandler := cache.ResourceEventHandlerFuncs{
AddFunc: t.resourceSliceAdd(ctx),
UpdateFunc: t.resourceSliceUpdate(ctx),
DeleteFunc: t.resourceSliceDelete(ctx),
}
t.resourceSlicesHandle, err = t.resourceSlices.AddEventHandler(sliceHandler)
if err != nil {
return fmt.Errorf("add event handler for ResourceSlices: %w", err)
}
taintHandler := cache.ResourceEventHandlerFuncs{
AddFunc: t.deviceTaintAdd(ctx),
UpdateFunc: t.deviceTaintUpdate(ctx),
DeleteFunc: t.deviceTaintDelete(ctx),
}
t.deviceTaintsHandle, err = t.deviceTaints.AddEventHandler(taintHandler)
if err != nil {
return fmt.Errorf("add event handler for DeviceTaintRules: %w", err)
}
classHandler := cache.ResourceEventHandlerFuncs{
AddFunc: t.deviceClassAdd(ctx),
UpdateFunc: t.deviceClassUpdate(ctx),
DeleteFunc: t.deviceClassDelete(ctx),
}
t.deviceClassesHandle, err = t.deviceClasses.AddEventHandler(classHandler)
if err != nil {
return fmt.Errorf("add event handler for DeviceClasses: %w", err)
}
return nil
}
// HasSynced returns true if the tracker is done with processing all
// currently existing input objects. Adding a new event handler at that
// point is possible and will emit events with up-to-date ResourceSlice
// objects.
func (t *Tracker) HasSynced() bool {
if !t.enableDeviceTaints {
return t.resourceSlices.HasSynced()
}
if t.resourceSlicesHandle != nil && !t.resourceSlicesHandle.HasSynced() {
return false
}
if t.deviceTaintsHandle != nil && !t.deviceTaintsHandle.HasSynced() {
return false
}
if t.deviceClassesHandle != nil && !t.deviceClassesHandle.HasSynced() {
return false
}
return true
}
// Stop ends all background activity and blocks until that shutdown is complete.
func (t *Tracker) Stop() {
if !t.enableDeviceTaints {
return
}
if t.broadcaster != nil {
t.broadcaster.Shutdown()
}
_ = t.resourceSlices.RemoveEventHandler(t.resourceSlicesHandle)
_ = t.deviceTaints.RemoveEventHandler(t.deviceTaintsHandle)
_ = t.deviceClasses.RemoveEventHandler(t.deviceClassesHandle)
}
// ListPatchedResourceSlices returns all ResourceSlices in the cluster with
// modifications from DeviceTaints applied.
func (t *Tracker) ListPatchedResourceSlices() ([]*resourceapi.ResourceSlice, error) {
if !t.enableDeviceTaints {
return t.resourceSliceLister.List(labels.Everything())
}
return typedSlice[*resourceapi.ResourceSlice](t.patchedResourceSlices.List()), nil
}
// AddEventHandler adds an event handler to the tracker. Events to a
// single handler are delivered sequentially, but there is no
// coordination between different handlers. A handler may use the
// tracker.
//
// The return value can be used to wait for cache synchronization.
// All currently know ResourceSlices get delivered via Add events
// before this method returns.
func (t *Tracker) AddEventHandler(handler cache.ResourceEventHandler) (cache.ResourceEventHandlerRegistration, error) {
if !t.enableDeviceTaints {
return t.resourceSlices.AddEventHandler(handler)
}
defer t.emitEvents()
t.rwMutex.Lock()
defer t.rwMutex.Unlock()
t.eventHandlers = append(t.eventHandlers, handler)
allObjs, _ := t.ListPatchedResourceSlices()
for _, obj := range allObjs {
t.eventQueue.Push(func() {
handler.OnAdd(obj, true)
})
}
// The tracker itself provides HasSynced for all registered event handlers.
// We don't support removal, so returning the same handle here for all
// of them is fine.
return t, nil
}
// emitEvents delivers all pending events that are in the queue, in the order
// in which they were stored there (FIFO).
func (t *Tracker) emitEvents() {
for {
t.rwMutex.Lock()
deliver, ok := t.eventQueue.Pop()
t.rwMutex.Unlock()
if !ok {
return
}
func() {
defer utilruntime.HandleCrash()
deliver()
}()
}
}
// pushEvent ensures that all currently registered event handlers get
// notified about a change when the caller starts delivering
// those with emitEvents.
//
// For a delete event, newObj is nil. For an add, oldObj is nil.
// An update has both as non-nil.
func (t *Tracker) pushEvent(oldObj, newObj any) {
t.rwMutex.Lock()
defer t.rwMutex.Unlock()
for _, handler := range t.eventHandlers {
handler := handler
if oldObj == nil {
t.eventQueue.Push(func() {
handler.OnAdd(newObj, false)
})
} else if newObj == nil {
t.eventQueue.Push(func() {
handler.OnDelete(oldObj)
})
} else {
t.eventQueue.Push(func() {
handler.OnUpdate(oldObj, newObj)
})
}
}
}
func sliceDriverPoolDeviceIndexFunc(obj any) ([]string, error) {
slice := obj.(*resourceapi.ResourceSlice)
drivers := []string{
anyDriver,
slice.Spec.Driver,
}
pools := []string{
anyPool,
slice.Spec.Pool.Name,
}
indexValues := make([]string, 0, len(drivers)*len(pools)*(1+len(slice.Spec.Devices)))
for _, driver := range drivers {
for _, pool := range pools {
indexValues = append(indexValues, deviceID(driver, pool, anyDevice))
for _, device := range slice.Spec.Devices {
indexValues = append(indexValues, deviceID(driver, pool, device.Name))
}
}
}
return indexValues, nil
}
func driverPoolDeviceIndexPatchKey(patch *resourcealphaapi.DeviceTaintRule) string {
deviceSelector := ptr.Deref(patch.Spec.DeviceSelector, resourcealphaapi.DeviceTaintSelector{})
driverKey := ptr.Deref(deviceSelector.Driver, anyDriver)
poolKey := ptr.Deref(deviceSelector.Pool, anyPool)
deviceKey := ptr.Deref(deviceSelector.Device, anyDevice)
return deviceID(driverKey, poolKey, deviceKey)
}
func (t *Tracker) sliceNamesForPatch(ctx context.Context, patch *resourcealphaapi.DeviceTaintRule) []string {
patchKey := driverPoolDeviceIndexPatchKey(patch)
sliceNames, err := t.resourceSlices.GetIndexer().IndexKeys(driverPoolDeviceIndexName, patchKey)
if err != nil {
t.handleError(ctx, err, "failed listing ResourceSlices for driver/pool/device key", "key", patchKey)
return nil
}
return sliceNames
}
func (t *Tracker) resourceSliceAdd(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
slice, ok := obj.(*resourceapi.ResourceSlice)
if !ok {
return
}
logger.V(5).Info("ResourceSlice add", "slice", klog.KObj(slice))
t.syncSlice(ctx, slice.Name, true)
}
}
func (t *Tracker) resourceSliceUpdate(ctx context.Context) func(oldObj, newObj any) {
logger := klog.FromContext(ctx)
return func(oldObj, newObj any) {
oldSlice, ok := oldObj.(*resourceapi.ResourceSlice)
if !ok {
return
}
newSlice, ok := newObj.(*resourceapi.ResourceSlice)
if !ok {
return
}
if loggerV := logger.V(6); loggerV.Enabled() {
// While debugging, one needs a full dump of the objects for context *and*
// a diff because otherwise small changes would be hard to spot.
loggerV.Info("ResourceSlice update", "slice", klog.Format(oldSlice), "oldSlice", klog.Format(newSlice), "diff", cmp.Diff(oldSlice, newSlice))
} else {
logger.V(5).Info("ResourceSlice update", "slice", klog.KObj(newSlice))
}
t.syncSlice(ctx, newSlice.Name, true)
}
}
func (t *Tracker) resourceSliceDelete(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
obj = tombstone.Obj
}
slice, ok := obj.(*resourceapi.ResourceSlice)
if !ok {
return
}
logger.V(5).Info("ResourceSlice delete", "slice", klog.KObj(slice))
t.syncSlice(ctx, slice.Name, true)
}
}
func (t *Tracker) deviceTaintAdd(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
patch, ok := obj.(*resourcealphaapi.DeviceTaintRule)
if !ok {
return
}
logger.V(5).Info("DeviceTaintRule add", "patch", klog.KObj(patch))
for _, sliceName := range t.sliceNamesForPatch(ctx, patch) {
t.syncSlice(ctx, sliceName, false)
}
}
}
func (t *Tracker) deviceTaintUpdate(ctx context.Context) func(oldObj, newObj any) {
logger := klog.FromContext(ctx)
return func(oldObj, newObj any) {
oldPatch, ok := oldObj.(*resourcealphaapi.DeviceTaintRule)
if !ok {
return
}
newPatch, ok := newObj.(*resourcealphaapi.DeviceTaintRule)
if !ok {
return
}
if loggerV := logger.V(6); loggerV.Enabled() {
loggerV.Info("DeviceTaintRule update", "patch", klog.KObj(newPatch), "diff", cmp.Diff(oldPatch, newPatch))
} else {
logger.V(5).Info("DeviceTaintRule update", "patch", klog.KObj(newPatch))
}
// Slices that matched the old patch may need to be updated, in
// case they no longer match the new patch and need to have the
// patch's changes reverted.
slicesToSync := sets.New[string]()
slicesToSync.Insert(t.sliceNamesForPatch(ctx, oldPatch)...)
slicesToSync.Insert(t.sliceNamesForPatch(ctx, newPatch)...)
for _, sliceName := range slicesToSync.UnsortedList() {
t.syncSlice(ctx, sliceName, false)
}
}
}
func (t *Tracker) deviceTaintDelete(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
obj = tombstone.Obj
}
patch, ok := obj.(*resourcealphaapi.DeviceTaintRule)
if !ok {
return
}
logger.V(5).Info("DeviceTaintRule delete", "patch", klog.KObj(patch))
for _, sliceName := range t.sliceNamesForPatch(ctx, patch) {
t.syncSlice(ctx, sliceName, false)
}
}
}
func (t *Tracker) deviceClassAdd(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
class, ok := obj.(*resourceapi.DeviceClass)
if !ok {
return
}
logger.V(5).Info("DeviceClass add", "class", klog.KObj(class))
for _, sliceName := range t.resourceSlices.GetIndexer().ListKeys() {
t.syncSlice(ctx, sliceName, false)
}
}
}
func (t *Tracker) deviceClassUpdate(ctx context.Context) func(oldObj, newObj any) {
logger := klog.FromContext(ctx)
return func(oldObj, newObj any) {
oldClass, ok := oldObj.(*resourceapi.DeviceClass)
if !ok {
return
}
newClass, ok := newObj.(*resourceapi.DeviceClass)
if !ok {
return
}
if loggerV := logger.V(6); loggerV.Enabled() {
loggerV.Info("DeviceClass update", "class", klog.KObj(newClass), "diff", cmp.Diff(oldClass, newClass))
} else {
logger.V(5).Info("DeviceClass update", "class", klog.KObj(newClass))
}
for _, sliceName := range t.resourceSlices.GetIndexer().ListKeys() {
t.syncSlice(ctx, sliceName, false)
}
}
}
func (t *Tracker) deviceClassDelete(ctx context.Context) func(obj any) {
logger := klog.FromContext(ctx)
return func(obj any) {
if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
obj = tombstone.Obj
}
class, ok := obj.(*resourceapi.ResourceSlice)
if !ok {
return
}
logger.V(5).Info("DeviceClass delete", "class", klog.KObj(class))
for _, sliceName := range t.resourceSlices.GetIndexer().ListKeys() {
t.syncSlice(ctx, sliceName, false)
}
}
}
// syncSlice updates the slice with the given name, applying
// DeviceTaints that match. sendEvent is used to force the Tracker
// to publish an event for listeners added by [Tracker.AddEventHandler]. It
// is set when syncSlice is triggered by a ResourceSlice event to avoid
// doing costly DeepEqual comparisons where possible.
func (t *Tracker) syncSlice(ctx context.Context, name string, sendEvent bool) {
defer t.emitEvents()
logger := klog.FromContext(ctx)
logger = klog.LoggerWithValues(logger, "resourceslice", name)
ctx = klog.NewContext(ctx, logger)
logger.V(5).Info("syncing ResourceSlice")
obj, sliceExists, err := t.resourceSlices.GetIndexer().GetByKey(name)
if err != nil {
t.handleError(ctx, err, "failed to lookup existing resource slice", "resourceslice", name)
return
}
oldPatchedObj, oldSliceExists, err := t.patchedResourceSlices.GetByKey(name)
if err != nil {
t.handleError(ctx, err, "failed to lookup cached patched resource slice", "resourceslice", name)
return
}
if !sliceExists {
err := t.patchedResourceSlices.Delete(oldPatchedObj)
if err != nil {
t.handleError(ctx, err, "failed to delete cached patched resource slice", "resourceslice", name)
return
}
t.pushEvent(oldPatchedObj, nil)
logger.V(5).Info("patched ResourceSlice deleted")
return
}
var oldPatchedSlice *resourceapi.ResourceSlice
if oldSliceExists {
var ok bool
oldPatchedSlice, ok = oldPatchedObj.(*resourceapi.ResourceSlice)
if !ok {
t.handleError(ctx, errors.New("invalid type in resource slice cache"), "expectedType", fmt.Sprintf("%T", (*resourceapi.ResourceSlice)(nil)), "gotType", fmt.Sprintf("%T", oldPatchedObj))
return
}
}
slice, ok := obj.(*resourceapi.ResourceSlice)
if !ok {
t.handleError(ctx, errors.New("invalid type in resource slice cache"), fmt.Sprintf("expected type to be %T, got %T", (*resourceapi.ResourceSlice)(nil), obj))
return
}
patches := typedSlice[*resourcealphaapi.DeviceTaintRule](t.deviceTaints.GetIndexer().List())
patchedSlice, err := t.applyPatches(ctx, slice, patches)
if err != nil {
t.handleError(ctx, err, "failed to apply patches to ResourceSlice", "resourceslice", klog.KObj(slice))
return
}
// When syncSlice is triggered by something other than a ResourceSlice
// event, only the device attributes and capacity might change. We
// deliberately avoid any costly DeepEqual-style comparisons here.
if !sendEvent && oldPatchedSlice != nil {
for i := range patchedSlice.Spec.Devices {
oldDevice := oldPatchedSlice.Spec.Devices[i]
newDevice := patchedSlice.Spec.Devices[i]
sendEvent = sendEvent ||
!slices.EqualFunc(getTaints(oldDevice), getTaints(newDevice), taintsEqual)
}
}
err = t.patchedResourceSlices.Add(patchedSlice)
if err != nil {
t.handleError(ctx, err, "failed to add patched resource slice to cache", "resourceslice", klog.KObj(patchedSlice))
return
}
if sendEvent {
t.pushEvent(oldPatchedObj, patchedSlice)
}
if loggerV := logger.V(6); loggerV.Enabled() {
loggerV.Info("ResourceSlice synced", "diff", cmp.Diff(oldPatchedObj, patchedSlice))
} else {
logger.V(5).Info("ResourceSlice synced")
}
}
func (t *Tracker) applyPatches(ctx context.Context, slice *resourceapi.ResourceSlice, taintRules []*resourcealphaapi.DeviceTaintRule) (*resourceapi.ResourceSlice, error) {
logger := klog.FromContext(ctx)
// slice will be DeepCopied just-in-time, only when necessary.
patchedSlice := slice
for _, taintRule := range taintRules {
logger := klog.LoggerWithValues(logger, "deviceTaintRule", klog.KObj(taintRule))
logger.V(6).Info("processing DeviceTaintRule")
deviceSelector := taintRule.Spec.DeviceSelector
var deviceClassExprs []cel.CompilationResult
var selectorExprs []cel.CompilationResult
var deviceName *string
if deviceSelector != nil {
if deviceSelector.Driver != nil && *deviceSelector.Driver != slice.Spec.Driver {
logger.V(7).Info("DeviceTaintRule does not apply, mismatched driver", "sliceDriver", slice.Spec.Driver, "taintDriver", *deviceSelector.Driver)
continue
}
if deviceSelector.Pool != nil && *deviceSelector.Pool != slice.Spec.Pool.Name {
logger.V(7).Info("DeviceTaintRule does not apply, mismatched pool", "slicePool", slice.Spec.Pool.Name, "taintPool", *deviceSelector.Pool)
continue
}
deviceName = deviceSelector.Device
if deviceSelector.DeviceClassName != nil {
logger := logger.WithValues("deviceClassName", *deviceSelector.DeviceClassName)
classObj, exists, err := t.deviceClasses.GetIndexer().GetByKey(*deviceSelector.DeviceClassName)
if err != nil {
return nil, fmt.Errorf("failed to get device class %s for DeviceTaintRule %s", *deviceSelector.DeviceClassName, taintRule.Name)
}
if !exists {
logger.V(7).Info("DeviceTaintRule does not apply, DeviceClass does not exist")
continue
}
class := classObj.(*resourceapi.DeviceClass)
for _, selector := range class.Spec.Selectors {
if selector.CEL != nil {
expr := t.celCache.GetOrCompile(selector.CEL.Expression)
deviceClassExprs = append(deviceClassExprs, expr)
}
}
}
for _, selector := range deviceSelector.Selectors {
if selector.CEL != nil {
expr := t.celCache.GetOrCompile(selector.CEL.Expression)
selectorExprs = append(selectorExprs, expr)
}
}
}
devices:
for dIndex, device := range slice.Spec.Devices {
deviceID := deviceID(slice.Spec.Driver, slice.Spec.Pool.Name, device.Name)
logger := logger.WithValues("device", deviceID)
if deviceName != nil && *deviceName != device.Name {
logger.V(7).Info("DeviceTaintRule does not apply, mismatched device", "sliceDevice", device.Name, "taintDevice", *deviceSelector.Device)
continue
}
deviceAttributes := getAttributes(device)
deviceCapacity := getCapacity(device)
for i, expr := range deviceClassExprs {
if expr.Error != nil {
// Could happen if some future apiserver accepted some
// future expression and then got downgraded. Normally
// the "stored expression" mechanism prevents that, but
// this code here might be more than one release older
// than the cluster it runs in.
return nil, fmt.Errorf("DeviceTaintRule %s: class %s: selector #%d: CEL compile error: %w", taintRule.Name, *deviceSelector.DeviceClassName, i, expr.Error)
}
matches, details, err := expr.DeviceMatches(ctx, cel.Device{Driver: slice.Spec.Driver, Attributes: deviceAttributes, Capacity: deviceCapacity})
logger.V(7).Info("CEL result", "class", *deviceSelector.DeviceClassName, "selector", i, "expression", expr.Expression, "matches", matches, "actualCost", ptr.Deref(details.ActualCost(), 0), "err", err)
if err != nil {
continue devices
}
if !matches {
continue devices
}
}
for i, expr := range selectorExprs {
if expr.Error != nil {
// Could happen if some future apiserver accepted some
// future expression and then got downgraded. Normally
// the "stored expression" mechanism prevents that, but
// this code here might be more than one release older
// than the cluster it runs in.
return nil, fmt.Errorf("DeviceTaintRule %s: selector #%d: CEL compile error: %w", taintRule.Name, i, expr.Error)
}
matches, details, err := expr.DeviceMatches(ctx, cel.Device{Driver: slice.Spec.Driver, Attributes: deviceAttributes, Capacity: deviceCapacity})
logger.V(7).Info("CEL result", "selector", i, "expression", expr.Expression, "matches", matches, "actualCost", ptr.Deref(details.ActualCost(), 0), "err", err)
if err != nil {
if t.recorder != nil {
t.recorder.Eventf(taintRule, v1.EventTypeWarning, "CELRuntimeError", "selector #%d: runtime error: %v", i, err)
}
continue devices
}
if !matches {
continue devices
}
}
logger.V(6).Info("applying matching DeviceTaintRule")
// TODO: remove conversion once taint is already in the right API package.
ta := resourceapi.DeviceTaint{
Key: taintRule.Spec.Taint.Key,
Value: taintRule.Spec.Taint.Value,
Effect: resourceapi.DeviceTaintEffect(taintRule.Spec.Taint.Effect),
TimeAdded: taintRule.Spec.Taint.TimeAdded,
}
if patchedSlice == slice {
patchedSlice = slice.DeepCopy()
}
appendTaint(&patchedSlice.Spec.Devices[dIndex], ta)
}
}
return patchedSlice, nil
}
func getAttributes(device resourceapi.Device) map[resourceapi.QualifiedName]resourceapi.DeviceAttribute {
if device.Basic != nil {
return device.Basic.Attributes
}
return nil
}
func getCapacity(device resourceapi.Device) map[resourceapi.QualifiedName]resourceapi.DeviceCapacity {
if device.Basic != nil {
return device.Basic.Capacity
}
return nil
}
func getTaints(device resourceapi.Device) []resourceapi.DeviceTaint {
if device.Basic != nil {
return device.Basic.Taints
}
return nil
}
func appendTaint(device *resourceapi.Device, taint resourceapi.DeviceTaint) {
if device.Basic != nil {
device.Basic.Taints = append(device.Basic.Taints, taint)
return
}
}
func taintsEqual(a, b resourceapi.DeviceTaint) bool {
return a.Key == b.Key &&
a.Effect == b.Effect &&
a.Value == b.Value &&
a.TimeAdded.Equal(b.TimeAdded) // Equal deals with nil.
}
func deviceID(driver, pool, device string) string {
return driver + "/" + pool + "/" + device
}
func typedSlice[T any](objs []any) []T {
if objs == nil {
return nil
}
typed := make([]T, 0, len(objs))
for _, obj := range objs {
typed = append(typed, obj.(T))
}
return typed
}

File diff suppressed because it is too large Load Diff

View File

@ -27,40 +27,71 @@ import (
draapi "k8s.io/dynamic-resource-allocation/api"
)
func nodeMatches(node *v1.Node, nodeNameToMatch string, allNodesMatch bool, nodeSelector *v1.NodeSelector) (bool, error) {
switch {
case nodeNameToMatch != "":
return node != nil && node.Name == nodeNameToMatch, nil
case allNodesMatch:
return true, nil
case nodeSelector != nil:
selector, err := nodeaffinity.NewNodeSelector(nodeSelector)
if err != nil {
return false, fmt.Errorf("failed to parse node selector %s: %w", nodeSelector.String(), err)
}
return selector.Match(node), nil
}
return false, nil
}
// GatherPools collects information about all resource pools which provide
// devices that are accessible from the given node.
//
// Out-dated slices are silently ignored. Pools may be incomplete (not all
// required slices available) or invalid (for example, device names not unique).
// Both is recorded in the result.
func GatherPools(ctx context.Context, slices []*resourceapi.ResourceSlice, node *v1.Node) ([]*Pool, error) {
func GatherPools(ctx context.Context, slices []*resourceapi.ResourceSlice, node *v1.Node, features Features) ([]*Pool, error) {
pools := make(map[PoolID]*Pool)
nodeName := ""
if node != nil {
nodeName = node.Name
}
for _, slice := range slices {
if !features.PartitionableDevices && (len(slice.Spec.SharedCounters) > 0 || slice.Spec.PerDeviceNodeSelection != nil) {
continue
}
switch {
case slice.Spec.NodeName != "":
if slice.Spec.NodeName == nodeName {
case slice.Spec.NodeName != "" || slice.Spec.AllNodes || slice.Spec.NodeSelector != nil:
match, err := nodeMatches(node, slice.Spec.NodeName, slice.Spec.AllNodes, slice.Spec.NodeSelector)
if err != nil {
return nil, fmt.Errorf("failed to perform node selection for slice %s: %w", slice.Name, err)
}
if match {
if err := addSlice(pools, slice); err != nil {
return nil, fmt.Errorf("add node slice %s: %w", slice.Name, err)
return nil, fmt.Errorf("failed to add node slice %s: %w", slice.Name, err)
}
}
case slice.Spec.AllNodes:
if err := addSlice(pools, slice); err != nil {
return nil, fmt.Errorf("add cluster slice %s: %w", slice.Name, err)
}
case slice.Spec.NodeSelector != nil:
// TODO: move conversion into api.
selector, err := nodeaffinity.NewNodeSelector(slice.Spec.NodeSelector)
if err != nil {
return nil, fmt.Errorf("node selector in resource slice %s: %w", slice.Name, err)
}
if selector.Match(node) {
if err := addSlice(pools, slice); err != nil {
return nil, fmt.Errorf("add matching slice %s: %w", slice.Name, err)
case slice.Spec.PerDeviceNodeSelection != nil && *slice.Spec.PerDeviceNodeSelection:
for _, device := range slice.Spec.Devices {
if device.Basic == nil {
continue
}
var nodeName string
var allNodes bool
if device.Basic.NodeName != nil {
nodeName = *device.Basic.NodeName
}
if device.Basic.AllNodes != nil {
allNodes = *device.Basic.AllNodes
}
match, err := nodeMatches(node, nodeName, allNodes, device.Basic.NodeSelector)
if err != nil {
return nil, fmt.Errorf("failed to perform node selection for device %s in slice %s: %w",
device.String(), slice.Name, err)
}
if match {
if err := addSlice(pools, slice); err != nil {
return nil, fmt.Errorf("failed to add node slice %s: %w", slice.Name, err)
}
break
}
}
default: