mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-14 18:53:35 +00:00
rebase: update K8s packages to v0.32.1
Update K8s packages in go.mod to v0.32.1 Signed-off-by: Praveen M <m.praveen@ibm.com>
This commit is contained in:
68
vendor/k8s.io/kubernetes/pkg/kubelet/config/apiserver.go
generated
vendored
Normal file
68
vendor/k8s.io/kubernetes/pkg/kubelet/config/apiserver.go
generated
vendored
Normal file
@ -0,0 +1,68 @@
|
||||
/*
|
||||
Copyright 2015 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/klog/v2"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
)
|
||||
|
||||
// WaitForAPIServerSyncPeriod is the period between checks for the node list/watch initial sync
|
||||
const WaitForAPIServerSyncPeriod = 1 * time.Second
|
||||
|
||||
// NewSourceApiserver creates a config source that watches and pulls from the apiserver.
|
||||
func NewSourceApiserver(c clientset.Interface, nodeName types.NodeName, nodeHasSynced func() bool, updates chan<- interface{}) {
|
||||
lw := cache.NewListWatchFromClient(c.CoreV1().RESTClient(), "pods", metav1.NamespaceAll, fields.OneTermEqualSelector("spec.nodeName", string(nodeName)))
|
||||
|
||||
// The Reflector responsible for watching pods at the apiserver should be run only after
|
||||
// the node sync with the apiserver has completed.
|
||||
klog.InfoS("Waiting for node sync before watching apiserver pods")
|
||||
go func() {
|
||||
for {
|
||||
if nodeHasSynced() {
|
||||
klog.V(4).InfoS("node sync completed")
|
||||
break
|
||||
}
|
||||
time.Sleep(WaitForAPIServerSyncPeriod)
|
||||
klog.V(4).InfoS("node sync has not completed yet")
|
||||
}
|
||||
klog.InfoS("Watching apiserver")
|
||||
newSourceApiserverFromLW(lw, updates)
|
||||
}()
|
||||
}
|
||||
|
||||
// newSourceApiserverFromLW holds creates a config source that watches and pulls from the apiserver.
|
||||
func newSourceApiserverFromLW(lw cache.ListerWatcher, updates chan<- interface{}) {
|
||||
send := func(objs []interface{}) {
|
||||
var pods []*v1.Pod
|
||||
for _, o := range objs {
|
||||
pods = append(pods, o.(*v1.Pod))
|
||||
}
|
||||
updates <- kubetypes.PodUpdate{Pods: pods, Op: kubetypes.SET, Source: kubetypes.ApiserverSource}
|
||||
}
|
||||
r := cache.NewReflector(lw, &v1.Pod{}, cache.NewUndeltaStore(send, cache.MetaNamespaceKeyFunc), 0)
|
||||
go r.Run(wait.NeverStop)
|
||||
}
|
191
vendor/k8s.io/kubernetes/pkg/kubelet/config/common.go
generated
vendored
Normal file
191
vendor/k8s.io/kubernetes/pkg/kubelet/config/common.go
generated
vendored
Normal file
@ -0,0 +1,191 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
"k8s.io/kubernetes/pkg/apis/core/helper"
|
||||
|
||||
// TODO: remove this import if
|
||||
// api.Registry.GroupOrDie(v1.GroupName).GroupVersion.String() is changed
|
||||
// to "v1"?
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
// Ensure that core apis are installed
|
||||
_ "k8s.io/kubernetes/pkg/apis/core/install"
|
||||
k8s_api_v1 "k8s.io/kubernetes/pkg/apis/core/v1"
|
||||
"k8s.io/kubernetes/pkg/apis/core/validation"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
"k8s.io/kubernetes/pkg/util/hash"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
maxConfigLength = 10 * 1 << 20 // 10MB
|
||||
)
|
||||
|
||||
// Generate a pod name that is unique among nodes by appending the nodeName.
|
||||
func generatePodName(name string, nodeName types.NodeName) string {
|
||||
return fmt.Sprintf("%s-%s", name, strings.ToLower(string(nodeName)))
|
||||
}
|
||||
|
||||
func applyDefaults(pod *api.Pod, source string, isFile bool, nodeName types.NodeName) error {
|
||||
if len(pod.UID) == 0 {
|
||||
hasher := md5.New()
|
||||
hash.DeepHashObject(hasher, pod)
|
||||
// DeepHashObject resets the hash, so we should write the pod source
|
||||
// information AFTER it.
|
||||
if isFile {
|
||||
fmt.Fprintf(hasher, "host:%s", nodeName)
|
||||
fmt.Fprintf(hasher, "file:%s", source)
|
||||
} else {
|
||||
fmt.Fprintf(hasher, "url:%s", source)
|
||||
}
|
||||
pod.UID = types.UID(hex.EncodeToString(hasher.Sum(nil)[0:]))
|
||||
klog.V(5).InfoS("Generated UID", "pod", klog.KObj(pod), "podUID", pod.UID, "source", source)
|
||||
}
|
||||
|
||||
pod.Name = generatePodName(pod.Name, nodeName)
|
||||
klog.V(5).InfoS("Generated pod name", "pod", klog.KObj(pod), "podUID", pod.UID, "source", source)
|
||||
|
||||
if pod.Namespace == "" {
|
||||
pod.Namespace = metav1.NamespaceDefault
|
||||
}
|
||||
klog.V(5).InfoS("Set namespace for pod", "pod", klog.KObj(pod), "source", source)
|
||||
|
||||
// Set the Host field to indicate this pod is scheduled on the current node.
|
||||
pod.Spec.NodeName = string(nodeName)
|
||||
|
||||
if pod.Annotations == nil {
|
||||
pod.Annotations = make(map[string]string)
|
||||
}
|
||||
// The generated UID is the hash of the file.
|
||||
pod.Annotations[kubetypes.ConfigHashAnnotationKey] = string(pod.UID)
|
||||
|
||||
if isFile {
|
||||
// Applying the default Taint tolerations to static pods,
|
||||
// so they are not evicted when there are node problems.
|
||||
helper.AddOrUpdateTolerationInPod(pod, &api.Toleration{
|
||||
Operator: "Exists",
|
||||
Effect: api.TaintEffectNoExecute,
|
||||
})
|
||||
}
|
||||
|
||||
// Set the default status to pending.
|
||||
pod.Status.Phase = api.PodPending
|
||||
return nil
|
||||
}
|
||||
|
||||
type defaultFunc func(pod *api.Pod) error
|
||||
|
||||
// A static pod tried to use a ClusterTrustBundle projected volume source.
|
||||
var ErrStaticPodTriedToUseClusterTrustBundle = errors.New("static pods may not use ClusterTrustBundle projected volume sources")
|
||||
|
||||
// tryDecodeSinglePod takes data and tries to extract valid Pod config information from it.
|
||||
func tryDecodeSinglePod(data []byte, defaultFn defaultFunc) (parsed bool, pod *v1.Pod, err error) {
|
||||
// JSON is valid YAML, so this should work for everything.
|
||||
json, err := utilyaml.ToJSON(data)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), json)
|
||||
if err != nil {
|
||||
return false, pod, err
|
||||
}
|
||||
|
||||
newPod, ok := obj.(*api.Pod)
|
||||
// Check whether the object could be converted to single pod.
|
||||
if !ok {
|
||||
return false, pod, fmt.Errorf("invalid pod: %#v", obj)
|
||||
}
|
||||
|
||||
if newPod.Name == "" {
|
||||
return true, pod, fmt.Errorf("invalid pod: name is needed for the pod")
|
||||
}
|
||||
|
||||
// Apply default values and validate the pod.
|
||||
if err = defaultFn(newPod); err != nil {
|
||||
return true, pod, err
|
||||
}
|
||||
if errs := validation.ValidatePodCreate(newPod, validation.PodValidationOptions{}); len(errs) > 0 {
|
||||
return true, pod, fmt.Errorf("invalid pod: %v", errs)
|
||||
}
|
||||
v1Pod := &v1.Pod{}
|
||||
if err := k8s_api_v1.Convert_core_Pod_To_v1_Pod(newPod, v1Pod, nil); err != nil {
|
||||
klog.ErrorS(err, "Pod failed to convert to v1", "pod", klog.KObj(newPod))
|
||||
return true, nil, err
|
||||
}
|
||||
|
||||
for _, v := range v1Pod.Spec.Volumes {
|
||||
if v.Projected == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, s := range v.Projected.Sources {
|
||||
if s.ClusterTrustBundle != nil {
|
||||
return true, nil, ErrStaticPodTriedToUseClusterTrustBundle
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true, v1Pod, nil
|
||||
}
|
||||
|
||||
func tryDecodePodList(data []byte, defaultFn defaultFunc) (parsed bool, pods v1.PodList, err error) {
|
||||
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data)
|
||||
if err != nil {
|
||||
return false, pods, err
|
||||
}
|
||||
|
||||
newPods, ok := obj.(*api.PodList)
|
||||
// Check whether the object could be converted to list of pods.
|
||||
if !ok {
|
||||
err = fmt.Errorf("invalid pods list: %#v", obj)
|
||||
return false, pods, err
|
||||
}
|
||||
|
||||
// Apply default values and validate pods.
|
||||
for i := range newPods.Items {
|
||||
newPod := &newPods.Items[i]
|
||||
if newPod.Name == "" {
|
||||
return true, pods, fmt.Errorf("invalid pod: name is needed for the pod")
|
||||
}
|
||||
if err = defaultFn(newPod); err != nil {
|
||||
return true, pods, err
|
||||
}
|
||||
if errs := validation.ValidatePodCreate(newPod, validation.PodValidationOptions{}); len(errs) > 0 {
|
||||
err = fmt.Errorf("invalid pod: %v", errs)
|
||||
return true, pods, err
|
||||
}
|
||||
}
|
||||
v1Pods := &v1.PodList{}
|
||||
if err := k8s_api_v1.Convert_core_PodList_To_v1_PodList(newPods, v1Pods, nil); err != nil {
|
||||
return true, pods, err
|
||||
}
|
||||
return true, *v1Pods, err
|
||||
}
|
499
vendor/k8s.io/kubernetes/pkg/kubelet/config/config.go
generated
vendored
Normal file
499
vendor/k8s.io/kubernetes/pkg/kubelet/config/config.go
generated
vendored
Normal file
@ -0,0 +1,499 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
"k8s.io/client-go/tools/record"
|
||||
"k8s.io/klog/v2"
|
||||
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
|
||||
"k8s.io/kubernetes/pkg/kubelet/events"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
"k8s.io/kubernetes/pkg/kubelet/util/format"
|
||||
)
|
||||
|
||||
// PodConfigNotificationMode describes how changes are sent to the update channel.
|
||||
type PodConfigNotificationMode int
|
||||
|
||||
const (
|
||||
// PodConfigNotificationUnknown is the default value for
|
||||
// PodConfigNotificationMode when uninitialized.
|
||||
PodConfigNotificationUnknown PodConfigNotificationMode = iota
|
||||
// PodConfigNotificationSnapshot delivers the full configuration as a SET whenever
|
||||
// any change occurs.
|
||||
PodConfigNotificationSnapshot
|
||||
// PodConfigNotificationSnapshotAndUpdates delivers an UPDATE and DELETE message whenever pods are
|
||||
// changed, and a SET message if there are any additions or removals.
|
||||
PodConfigNotificationSnapshotAndUpdates
|
||||
// PodConfigNotificationIncremental delivers ADD, UPDATE, DELETE, REMOVE, RECONCILE to the update channel.
|
||||
PodConfigNotificationIncremental
|
||||
)
|
||||
|
||||
type podStartupSLIObserver interface {
|
||||
ObservedPodOnWatch(pod *v1.Pod, when time.Time)
|
||||
}
|
||||
|
||||
// PodConfig is a configuration mux that merges many sources of pod configuration into a single
|
||||
// consistent structure, and then delivers incremental change notifications to listeners
|
||||
// in order.
|
||||
type PodConfig struct {
|
||||
pods *podStorage
|
||||
mux *mux
|
||||
|
||||
// the channel of denormalized changes passed to listeners
|
||||
updates chan kubetypes.PodUpdate
|
||||
|
||||
// contains the list of all configured sources
|
||||
sourcesLock sync.Mutex
|
||||
sources sets.Set[string]
|
||||
}
|
||||
|
||||
// NewPodConfig creates an object that can merge many configuration sources into a stream
|
||||
// of normalized updates to a pod configuration.
|
||||
func NewPodConfig(mode PodConfigNotificationMode, recorder record.EventRecorder, startupSLIObserver podStartupSLIObserver) *PodConfig {
|
||||
updates := make(chan kubetypes.PodUpdate, 50)
|
||||
storage := newPodStorage(updates, mode, recorder, startupSLIObserver)
|
||||
podConfig := &PodConfig{
|
||||
pods: storage,
|
||||
mux: newMux(storage),
|
||||
updates: updates,
|
||||
sources: sets.Set[string]{},
|
||||
}
|
||||
return podConfig
|
||||
}
|
||||
|
||||
// Channel creates or returns a config source channel. The channel
|
||||
// only accepts PodUpdates
|
||||
func (c *PodConfig) Channel(ctx context.Context, source string) chan<- interface{} {
|
||||
c.sourcesLock.Lock()
|
||||
defer c.sourcesLock.Unlock()
|
||||
c.sources.Insert(source)
|
||||
return c.mux.ChannelWithContext(ctx, source)
|
||||
}
|
||||
|
||||
// SeenAllSources returns true if seenSources contains all sources in the
|
||||
// config, and also this config has received a SET message from each source.
|
||||
func (c *PodConfig) SeenAllSources(seenSources sets.Set[string]) bool {
|
||||
if c.pods == nil {
|
||||
return false
|
||||
}
|
||||
c.sourcesLock.Lock()
|
||||
defer c.sourcesLock.Unlock()
|
||||
klog.V(5).InfoS("Looking for sources, have seen", "sources", sets.List(c.sources), "seenSources", seenSources)
|
||||
return seenSources.HasAll(sets.List(c.sources)...) && c.pods.seenSources(sets.List(c.sources)...)
|
||||
}
|
||||
|
||||
// Updates returns a channel of updates to the configuration, properly denormalized.
|
||||
func (c *PodConfig) Updates() <-chan kubetypes.PodUpdate {
|
||||
return c.updates
|
||||
}
|
||||
|
||||
// Sync requests the full configuration be delivered to the update channel.
|
||||
func (c *PodConfig) Sync() {
|
||||
c.pods.sync()
|
||||
}
|
||||
|
||||
// podStorage manages the current pod state at any point in time and ensures updates
|
||||
// to the channel are delivered in order. Note that this object is an in-memory source of
|
||||
// "truth" and on creation contains zero entries. Once all previously read sources are
|
||||
// available, then this object should be considered authoritative.
|
||||
type podStorage struct {
|
||||
podLock sync.RWMutex
|
||||
// map of source name to pod uid to pod reference
|
||||
pods map[string]map[types.UID]*v1.Pod
|
||||
mode PodConfigNotificationMode
|
||||
|
||||
// ensures that updates are delivered in strict order
|
||||
// on the updates channel
|
||||
updateLock sync.Mutex
|
||||
updates chan<- kubetypes.PodUpdate
|
||||
|
||||
// contains the set of all sources that have sent at least one SET
|
||||
sourcesSeenLock sync.RWMutex
|
||||
sourcesSeen sets.Set[string]
|
||||
|
||||
// the EventRecorder to use
|
||||
recorder record.EventRecorder
|
||||
|
||||
startupSLIObserver podStartupSLIObserver
|
||||
}
|
||||
|
||||
// TODO: PodConfigNotificationMode could be handled by a listener to the updates channel
|
||||
// in the future, especially with multiple listeners.
|
||||
// TODO: allow initialization of the current state of the store with snapshotted version.
|
||||
func newPodStorage(updates chan<- kubetypes.PodUpdate, mode PodConfigNotificationMode, recorder record.EventRecorder, startupSLIObserver podStartupSLIObserver) *podStorage {
|
||||
return &podStorage{
|
||||
pods: make(map[string]map[types.UID]*v1.Pod),
|
||||
mode: mode,
|
||||
updates: updates,
|
||||
sourcesSeen: sets.Set[string]{},
|
||||
recorder: recorder,
|
||||
startupSLIObserver: startupSLIObserver,
|
||||
}
|
||||
}
|
||||
|
||||
// Merge normalizes a set of incoming changes from different sources into a map of all Pods
|
||||
// and ensures that redundant changes are filtered out, and then pushes zero or more minimal
|
||||
// updates onto the update channel. Ensures that updates are delivered in order.
|
||||
func (s *podStorage) Merge(source string, change interface{}) error {
|
||||
s.updateLock.Lock()
|
||||
defer s.updateLock.Unlock()
|
||||
|
||||
seenBefore := s.sourcesSeen.Has(source)
|
||||
adds, updates, deletes, removes, reconciles := s.merge(source, change)
|
||||
firstSet := !seenBefore && s.sourcesSeen.Has(source)
|
||||
|
||||
// deliver update notifications
|
||||
switch s.mode {
|
||||
case PodConfigNotificationIncremental:
|
||||
if len(removes.Pods) > 0 {
|
||||
s.updates <- *removes
|
||||
}
|
||||
if len(adds.Pods) > 0 {
|
||||
s.updates <- *adds
|
||||
}
|
||||
if len(updates.Pods) > 0 {
|
||||
s.updates <- *updates
|
||||
}
|
||||
if len(deletes.Pods) > 0 {
|
||||
s.updates <- *deletes
|
||||
}
|
||||
if firstSet && len(adds.Pods) == 0 && len(updates.Pods) == 0 && len(deletes.Pods) == 0 {
|
||||
// Send an empty update when first seeing the source and there are
|
||||
// no ADD or UPDATE or DELETE pods from the source. This signals kubelet that
|
||||
// the source is ready.
|
||||
s.updates <- *adds
|
||||
}
|
||||
// Only add reconcile support here, because kubelet doesn't support Snapshot update now.
|
||||
if len(reconciles.Pods) > 0 {
|
||||
s.updates <- *reconciles
|
||||
}
|
||||
|
||||
case PodConfigNotificationSnapshotAndUpdates:
|
||||
if len(removes.Pods) > 0 || len(adds.Pods) > 0 || firstSet {
|
||||
s.updates <- kubetypes.PodUpdate{Pods: s.mergedState().([]*v1.Pod), Op: kubetypes.SET, Source: source}
|
||||
}
|
||||
if len(updates.Pods) > 0 {
|
||||
s.updates <- *updates
|
||||
}
|
||||
if len(deletes.Pods) > 0 {
|
||||
s.updates <- *deletes
|
||||
}
|
||||
|
||||
case PodConfigNotificationSnapshot:
|
||||
if len(updates.Pods) > 0 || len(deletes.Pods) > 0 || len(adds.Pods) > 0 || len(removes.Pods) > 0 || firstSet {
|
||||
s.updates <- kubetypes.PodUpdate{Pods: s.mergedState().([]*v1.Pod), Op: kubetypes.SET, Source: source}
|
||||
}
|
||||
|
||||
case PodConfigNotificationUnknown:
|
||||
fallthrough
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported PodConfigNotificationMode: %#v", s.mode))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *podStorage) merge(source string, change interface{}) (adds, updates, deletes, removes, reconciles *kubetypes.PodUpdate) {
|
||||
s.podLock.Lock()
|
||||
defer s.podLock.Unlock()
|
||||
|
||||
addPods := []*v1.Pod{}
|
||||
updatePods := []*v1.Pod{}
|
||||
deletePods := []*v1.Pod{}
|
||||
removePods := []*v1.Pod{}
|
||||
reconcilePods := []*v1.Pod{}
|
||||
|
||||
pods := s.pods[source]
|
||||
if pods == nil {
|
||||
pods = make(map[types.UID]*v1.Pod)
|
||||
}
|
||||
|
||||
// updatePodFunc is the local function which updates the pod cache *oldPods* with new pods *newPods*.
|
||||
// After updated, new pod will be stored in the pod cache *pods*.
|
||||
// Notice that *pods* and *oldPods* could be the same cache.
|
||||
updatePodsFunc := func(newPods []*v1.Pod, oldPods, pods map[types.UID]*v1.Pod) {
|
||||
filtered := filterInvalidPods(newPods, source, s.recorder)
|
||||
for _, ref := range filtered {
|
||||
// Annotate the pod with the source before any comparison.
|
||||
if ref.Annotations == nil {
|
||||
ref.Annotations = make(map[string]string)
|
||||
}
|
||||
ref.Annotations[kubetypes.ConfigSourceAnnotationKey] = source
|
||||
// ignore static pods
|
||||
if !kubetypes.IsStaticPod(ref) {
|
||||
s.startupSLIObserver.ObservedPodOnWatch(ref, time.Now())
|
||||
}
|
||||
if existing, found := oldPods[ref.UID]; found {
|
||||
pods[ref.UID] = existing
|
||||
needUpdate, needReconcile, needGracefulDelete := checkAndUpdatePod(existing, ref)
|
||||
if needUpdate {
|
||||
updatePods = append(updatePods, existing)
|
||||
} else if needReconcile {
|
||||
reconcilePods = append(reconcilePods, existing)
|
||||
} else if needGracefulDelete {
|
||||
deletePods = append(deletePods, existing)
|
||||
}
|
||||
continue
|
||||
}
|
||||
recordFirstSeenTime(ref)
|
||||
pods[ref.UID] = ref
|
||||
addPods = append(addPods, ref)
|
||||
}
|
||||
}
|
||||
|
||||
update := change.(kubetypes.PodUpdate)
|
||||
switch update.Op {
|
||||
case kubetypes.ADD, kubetypes.UPDATE, kubetypes.DELETE:
|
||||
if update.Op == kubetypes.ADD {
|
||||
klog.V(4).InfoS("Adding new pods from source", "source", source, "pods", klog.KObjSlice(update.Pods))
|
||||
} else if update.Op == kubetypes.DELETE {
|
||||
klog.V(4).InfoS("Gracefully deleting pods from source", "source", source, "pods", klog.KObjSlice(update.Pods))
|
||||
} else {
|
||||
klog.V(4).InfoS("Updating pods from source", "source", source, "pods", klog.KObjSlice(update.Pods))
|
||||
}
|
||||
updatePodsFunc(update.Pods, pods, pods)
|
||||
|
||||
case kubetypes.REMOVE:
|
||||
klog.V(4).InfoS("Removing pods from source", "source", source, "pods", klog.KObjSlice(update.Pods))
|
||||
for _, value := range update.Pods {
|
||||
if existing, found := pods[value.UID]; found {
|
||||
// this is a delete
|
||||
delete(pods, value.UID)
|
||||
removePods = append(removePods, existing)
|
||||
continue
|
||||
}
|
||||
// this is a no-op
|
||||
}
|
||||
|
||||
case kubetypes.SET:
|
||||
klog.V(4).InfoS("Setting pods for source", "source", source)
|
||||
s.markSourceSet(source)
|
||||
// Clear the old map entries by just creating a new map
|
||||
oldPods := pods
|
||||
pods = make(map[types.UID]*v1.Pod)
|
||||
updatePodsFunc(update.Pods, oldPods, pods)
|
||||
for uid, existing := range oldPods {
|
||||
if _, found := pods[uid]; !found {
|
||||
// this is a delete
|
||||
removePods = append(removePods, existing)
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
klog.InfoS("Received invalid update type", "type", update)
|
||||
|
||||
}
|
||||
|
||||
s.pods[source] = pods
|
||||
|
||||
adds = &kubetypes.PodUpdate{Op: kubetypes.ADD, Pods: copyPods(addPods), Source: source}
|
||||
updates = &kubetypes.PodUpdate{Op: kubetypes.UPDATE, Pods: copyPods(updatePods), Source: source}
|
||||
deletes = &kubetypes.PodUpdate{Op: kubetypes.DELETE, Pods: copyPods(deletePods), Source: source}
|
||||
removes = &kubetypes.PodUpdate{Op: kubetypes.REMOVE, Pods: copyPods(removePods), Source: source}
|
||||
reconciles = &kubetypes.PodUpdate{Op: kubetypes.RECONCILE, Pods: copyPods(reconcilePods), Source: source}
|
||||
|
||||
return adds, updates, deletes, removes, reconciles
|
||||
}
|
||||
|
||||
func (s *podStorage) markSourceSet(source string) {
|
||||
s.sourcesSeenLock.Lock()
|
||||
defer s.sourcesSeenLock.Unlock()
|
||||
s.sourcesSeen.Insert(source)
|
||||
}
|
||||
|
||||
func (s *podStorage) seenSources(sources ...string) bool {
|
||||
s.sourcesSeenLock.RLock()
|
||||
defer s.sourcesSeenLock.RUnlock()
|
||||
return s.sourcesSeen.HasAll(sources...)
|
||||
}
|
||||
|
||||
func filterInvalidPods(pods []*v1.Pod, source string, recorder record.EventRecorder) (filtered []*v1.Pod) {
|
||||
names := sets.Set[string]{}
|
||||
for i, pod := range pods {
|
||||
// Pods from each source are assumed to have passed validation individually.
|
||||
// This function only checks if there is any naming conflict.
|
||||
name := kubecontainer.GetPodFullName(pod)
|
||||
if names.Has(name) {
|
||||
klog.InfoS("Pod failed validation due to duplicate pod name, ignoring", "index", i, "pod", klog.KObj(pod), "source", source)
|
||||
recorder.Eventf(pod, v1.EventTypeWarning, events.FailedValidation, "Error validating pod %s from %s due to duplicate pod name %q, ignoring", format.Pod(pod), source, pod.Name)
|
||||
continue
|
||||
} else {
|
||||
names.Insert(name)
|
||||
}
|
||||
|
||||
filtered = append(filtered, pod)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Annotations that the kubelet adds to the pod.
|
||||
var localAnnotations = []string{
|
||||
kubetypes.ConfigSourceAnnotationKey,
|
||||
kubetypes.ConfigMirrorAnnotationKey,
|
||||
kubetypes.ConfigFirstSeenAnnotationKey,
|
||||
}
|
||||
|
||||
func isLocalAnnotationKey(key string) bool {
|
||||
for _, localKey := range localAnnotations {
|
||||
if key == localKey {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isAnnotationMapEqual returns true if the existing annotation Map is equal to candidate except
|
||||
// for local annotations.
|
||||
func isAnnotationMapEqual(existingMap, candidateMap map[string]string) bool {
|
||||
if candidateMap == nil {
|
||||
candidateMap = make(map[string]string)
|
||||
}
|
||||
for k, v := range candidateMap {
|
||||
if isLocalAnnotationKey(k) {
|
||||
continue
|
||||
}
|
||||
if existingValue, ok := existingMap[k]; ok && existingValue == v {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
for k := range existingMap {
|
||||
if isLocalAnnotationKey(k) {
|
||||
continue
|
||||
}
|
||||
// stale entry in existing map.
|
||||
if _, exists := candidateMap[k]; !exists {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// recordFirstSeenTime records the first seen time of this pod.
|
||||
func recordFirstSeenTime(pod *v1.Pod) {
|
||||
klog.V(4).InfoS("Receiving a new pod", "pod", klog.KObj(pod))
|
||||
pod.Annotations[kubetypes.ConfigFirstSeenAnnotationKey] = kubetypes.NewTimestamp().GetString()
|
||||
}
|
||||
|
||||
// updateAnnotations returns an Annotation map containing the api annotation map plus
|
||||
// locally managed annotations
|
||||
func updateAnnotations(existing, ref *v1.Pod) {
|
||||
annotations := make(map[string]string, len(ref.Annotations)+len(localAnnotations))
|
||||
for k, v := range ref.Annotations {
|
||||
annotations[k] = v
|
||||
}
|
||||
for _, k := range localAnnotations {
|
||||
if v, ok := existing.Annotations[k]; ok {
|
||||
annotations[k] = v
|
||||
}
|
||||
}
|
||||
existing.Annotations = annotations
|
||||
}
|
||||
|
||||
func podsDifferSemantically(existing, ref *v1.Pod) bool {
|
||||
if reflect.DeepEqual(existing.Spec, ref.Spec) &&
|
||||
reflect.DeepEqual(existing.Labels, ref.Labels) &&
|
||||
reflect.DeepEqual(existing.DeletionTimestamp, ref.DeletionTimestamp) &&
|
||||
reflect.DeepEqual(existing.DeletionGracePeriodSeconds, ref.DeletionGracePeriodSeconds) &&
|
||||
isAnnotationMapEqual(existing.Annotations, ref.Annotations) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// checkAndUpdatePod updates existing, and:
|
||||
// - if ref makes a meaningful change, returns needUpdate=true
|
||||
// - if ref makes a meaningful change, and this change is graceful deletion, returns needGracefulDelete=true
|
||||
// - if ref makes no meaningful change, but changes the pod status, returns needReconcile=true
|
||||
// - else return all false
|
||||
// Now, needUpdate, needGracefulDelete and needReconcile should never be both true
|
||||
func checkAndUpdatePod(existing, ref *v1.Pod) (needUpdate, needReconcile, needGracefulDelete bool) {
|
||||
|
||||
// 1. this is a reconcile
|
||||
// TODO: it would be better to update the whole object and only preserve certain things
|
||||
// like the source annotation or the UID (to ensure safety)
|
||||
if !podsDifferSemantically(existing, ref) {
|
||||
// this is not an update
|
||||
// Only check reconcile when it is not an update, because if the pod is going to
|
||||
// be updated, an extra reconcile is unnecessary
|
||||
if !reflect.DeepEqual(existing.Status, ref.Status) {
|
||||
// Pod with changed pod status needs reconcile, because kubelet should
|
||||
// be the source of truth of pod status.
|
||||
existing.Status = ref.Status
|
||||
needReconcile = true
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Overwrite the first-seen time with the existing one. This is our own
|
||||
// internal annotation, there is no need to update.
|
||||
ref.Annotations[kubetypes.ConfigFirstSeenAnnotationKey] = existing.Annotations[kubetypes.ConfigFirstSeenAnnotationKey]
|
||||
|
||||
existing.Spec = ref.Spec
|
||||
existing.Labels = ref.Labels
|
||||
existing.DeletionTimestamp = ref.DeletionTimestamp
|
||||
existing.DeletionGracePeriodSeconds = ref.DeletionGracePeriodSeconds
|
||||
existing.Status = ref.Status
|
||||
updateAnnotations(existing, ref)
|
||||
|
||||
// 2. this is an graceful delete
|
||||
if ref.DeletionTimestamp != nil {
|
||||
needGracefulDelete = true
|
||||
} else {
|
||||
// 3. this is an update
|
||||
needUpdate = true
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// sync sends a copy of the current state through the update channel.
|
||||
func (s *podStorage) sync() {
|
||||
s.updateLock.Lock()
|
||||
defer s.updateLock.Unlock()
|
||||
s.updates <- kubetypes.PodUpdate{Pods: s.mergedState().([]*v1.Pod), Op: kubetypes.SET, Source: kubetypes.AllSource}
|
||||
}
|
||||
|
||||
func (s *podStorage) mergedState() interface{} {
|
||||
s.podLock.RLock()
|
||||
defer s.podLock.RUnlock()
|
||||
pods := make([]*v1.Pod, 0)
|
||||
for _, sourcePods := range s.pods {
|
||||
for _, podRef := range sourcePods {
|
||||
pods = append(pods, podRef.DeepCopy())
|
||||
}
|
||||
}
|
||||
return pods
|
||||
}
|
||||
|
||||
func copyPods(sourcePods []*v1.Pod) []*v1.Pod {
|
||||
pods := []*v1.Pod{}
|
||||
for _, source := range sourcePods {
|
||||
// Use a deep copy here just in case
|
||||
pods = append(pods, source.DeepCopy())
|
||||
}
|
||||
return pods
|
||||
}
|
33
vendor/k8s.io/kubernetes/pkg/kubelet/config/defaults.go
generated
vendored
Normal file
33
vendor/k8s.io/kubernetes/pkg/kubelet/config/defaults.go
generated
vendored
Normal file
@ -0,0 +1,33 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
// Defines sane defaults for the kubelet config.
|
||||
const (
|
||||
DefaultKubeletPodsDirName = "pods"
|
||||
DefaultKubeletVolumesDirName = "volumes"
|
||||
DefaultKubeletVolumeSubpathsDirName = "volume-subpaths"
|
||||
DefaultKubeletVolumeDevicesDirName = "volumeDevices"
|
||||
DefaultKubeletPluginsDirName = "plugins"
|
||||
DefaultKubeletPluginsRegistrationDirName = "plugins_registry"
|
||||
DefaultKubeletContainersDirName = "containers"
|
||||
DefaultKubeletPluginContainersDirName = "plugin-containers"
|
||||
DefaultKubeletPodResourcesDirName = "pod-resources"
|
||||
KubeletPluginsDirSELinuxLabel = "system_u:object_r:container_file_t:s0"
|
||||
KubeletContainersSharedSELinuxLabel = "system_u:object_r:container_file_t:s0"
|
||||
DefaultKubeletCheckpointsDirName = "checkpoints"
|
||||
)
|
18
vendor/k8s.io/kubernetes/pkg/kubelet/config/doc.go
generated
vendored
Normal file
18
vendor/k8s.io/kubernetes/pkg/kubelet/config/doc.go
generated
vendored
Normal file
@ -0,0 +1,18 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package config implements the pod configuration readers.
|
||||
package config // import "k8s.io/kubernetes/pkg/kubelet/config"
|
245
vendor/k8s.io/kubernetes/pkg/kubelet/config/file.go
generated
vendored
Normal file
245
vendor/k8s.io/kubernetes/pkg/kubelet/config/file.go
generated
vendored
Normal file
@ -0,0 +1,245 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
utilio "k8s.io/utils/io"
|
||||
)
|
||||
|
||||
type podEventType int
|
||||
|
||||
const (
|
||||
podAdd podEventType = iota
|
||||
podModify
|
||||
podDelete
|
||||
|
||||
eventBufferLen = 10
|
||||
)
|
||||
|
||||
type watchEvent struct {
|
||||
fileName string
|
||||
eventType podEventType
|
||||
}
|
||||
|
||||
type sourceFile struct {
|
||||
path string
|
||||
nodeName types.NodeName
|
||||
period time.Duration
|
||||
store cache.Store
|
||||
fileKeyMapping map[string]string
|
||||
updates chan<- interface{}
|
||||
watchEvents chan *watchEvent
|
||||
}
|
||||
|
||||
// NewSourceFile watches a config file for changes.
|
||||
func NewSourceFile(path string, nodeName types.NodeName, period time.Duration, updates chan<- interface{}) {
|
||||
// "github.com/sigma/go-inotify" requires a path without trailing "/"
|
||||
path = strings.TrimRight(path, string(os.PathSeparator))
|
||||
|
||||
config := newSourceFile(path, nodeName, period, updates)
|
||||
klog.V(1).InfoS("Watching path", "path", path)
|
||||
config.run()
|
||||
}
|
||||
|
||||
func newSourceFile(path string, nodeName types.NodeName, period time.Duration, updates chan<- interface{}) *sourceFile {
|
||||
send := func(objs []interface{}) {
|
||||
var pods []*v1.Pod
|
||||
for _, o := range objs {
|
||||
pods = append(pods, o.(*v1.Pod))
|
||||
}
|
||||
updates <- kubetypes.PodUpdate{Pods: pods, Op: kubetypes.SET, Source: kubetypes.FileSource}
|
||||
}
|
||||
store := cache.NewUndeltaStore(send, cache.MetaNamespaceKeyFunc)
|
||||
return &sourceFile{
|
||||
path: path,
|
||||
nodeName: nodeName,
|
||||
period: period,
|
||||
store: store,
|
||||
fileKeyMapping: map[string]string{},
|
||||
updates: updates,
|
||||
watchEvents: make(chan *watchEvent, eventBufferLen),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sourceFile) run() {
|
||||
listTicker := time.NewTicker(s.period)
|
||||
|
||||
go func() {
|
||||
// Read path immediately to speed up startup.
|
||||
if err := s.listConfig(); err != nil {
|
||||
klog.ErrorS(err, "Unable to read config path", "path", s.path)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-listTicker.C:
|
||||
if err := s.listConfig(); err != nil {
|
||||
klog.ErrorS(err, "Unable to read config path", "path", s.path)
|
||||
}
|
||||
case e := <-s.watchEvents:
|
||||
if err := s.consumeWatchEvent(e); err != nil {
|
||||
klog.ErrorS(err, "Unable to process watch event")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
s.startWatch()
|
||||
}
|
||||
|
||||
func (s *sourceFile) applyDefaults(pod *api.Pod, source string) error {
|
||||
return applyDefaults(pod, source, true, s.nodeName)
|
||||
}
|
||||
|
||||
func (s *sourceFile) listConfig() error {
|
||||
path := s.path
|
||||
statInfo, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Emit an update with an empty PodList to allow FileSource to be marked as seen
|
||||
s.updates <- kubetypes.PodUpdate{Pods: []*v1.Pod{}, Op: kubetypes.SET, Source: kubetypes.FileSource}
|
||||
return fmt.Errorf("path does not exist, ignoring")
|
||||
}
|
||||
|
||||
switch {
|
||||
case statInfo.Mode().IsDir():
|
||||
pods, err := s.extractFromDir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(pods) == 0 {
|
||||
// Emit an update with an empty PodList to allow FileSource to be marked as seen
|
||||
s.updates <- kubetypes.PodUpdate{Pods: pods, Op: kubetypes.SET, Source: kubetypes.FileSource}
|
||||
return nil
|
||||
}
|
||||
return s.replaceStore(pods...)
|
||||
|
||||
case statInfo.Mode().IsRegular():
|
||||
pod, err := s.extractFromFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.replaceStore(pod)
|
||||
|
||||
default:
|
||||
return fmt.Errorf("path is not a directory or file")
|
||||
}
|
||||
}
|
||||
|
||||
// Get as many pod manifests as we can from a directory. Return an error if and only if something
|
||||
// prevented us from reading anything at all. Do not return an error if only some files
|
||||
// were problematic.
|
||||
func (s *sourceFile) extractFromDir(name string) ([]*v1.Pod, error) {
|
||||
dirents, err := filepath.Glob(filepath.Join(name, "[^.]*"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("glob failed: %v", err)
|
||||
}
|
||||
|
||||
pods := make([]*v1.Pod, 0, len(dirents))
|
||||
if len(dirents) == 0 {
|
||||
return pods, nil
|
||||
}
|
||||
|
||||
sort.Strings(dirents)
|
||||
for _, path := range dirents {
|
||||
statInfo, err := os.Stat(path)
|
||||
if err != nil {
|
||||
klog.ErrorS(err, "Could not get metadata", "path", path)
|
||||
continue
|
||||
}
|
||||
|
||||
switch {
|
||||
case statInfo.Mode().IsDir():
|
||||
klog.ErrorS(nil, "Provided manifest path is a directory, not recursing into manifest path", "path", path)
|
||||
case statInfo.Mode().IsRegular():
|
||||
pod, err := s.extractFromFile(path)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
klog.ErrorS(err, "Could not process manifest file", "path", path)
|
||||
}
|
||||
} else {
|
||||
pods = append(pods, pod)
|
||||
}
|
||||
default:
|
||||
klog.ErrorS(nil, "Manifest path is not a directory or file", "path", path, "mode", statInfo.Mode())
|
||||
}
|
||||
}
|
||||
return pods, nil
|
||||
}
|
||||
|
||||
// extractFromFile parses a file for Pod configuration information.
|
||||
func (s *sourceFile) extractFromFile(filename string) (pod *v1.Pod, err error) {
|
||||
klog.V(3).InfoS("Reading config file", "path", filename)
|
||||
defer func() {
|
||||
if err == nil && pod != nil {
|
||||
objKey, keyErr := cache.MetaNamespaceKeyFunc(pod)
|
||||
if keyErr != nil {
|
||||
err = keyErr
|
||||
return
|
||||
}
|
||||
s.fileKeyMapping[filename] = objKey
|
||||
}
|
||||
}()
|
||||
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return pod, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
data, err := utilio.ReadAtMost(file, maxConfigLength)
|
||||
if err != nil {
|
||||
return pod, err
|
||||
}
|
||||
|
||||
defaultFn := func(pod *api.Pod) error {
|
||||
return s.applyDefaults(pod, filename)
|
||||
}
|
||||
|
||||
parsed, pod, podErr := tryDecodeSinglePod(data, defaultFn)
|
||||
if parsed {
|
||||
if podErr != nil {
|
||||
return pod, podErr
|
||||
}
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
return pod, fmt.Errorf("%v: couldn't parse as pod(%v), please check config file", filename, podErr)
|
||||
}
|
||||
|
||||
func (s *sourceFile) replaceStore(pods ...*v1.Pod) (err error) {
|
||||
objs := []interface{}{}
|
||||
for _, pod := range pods {
|
||||
objs = append(objs, pod)
|
||||
}
|
||||
return s.store.Replace(objs, "")
|
||||
}
|
154
vendor/k8s.io/kubernetes/pkg/kubelet/config/file_linux.go
generated
vendored
Normal file
154
vendor/k8s.io/kubernetes/pkg/kubelet/config/file_linux.go
generated
vendored
Normal file
@ -0,0 +1,154 @@
|
||||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/util/flowcontrol"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
)
|
||||
|
||||
const (
|
||||
retryPeriod = 1 * time.Second
|
||||
maxRetryPeriod = 20 * time.Second
|
||||
)
|
||||
|
||||
type retryableError struct {
|
||||
message string
|
||||
}
|
||||
|
||||
func (e *retryableError) Error() string {
|
||||
return e.message
|
||||
}
|
||||
|
||||
func (s *sourceFile) startWatch() {
|
||||
backOff := flowcontrol.NewBackOff(retryPeriod, maxRetryPeriod)
|
||||
backOffID := "watch"
|
||||
|
||||
go wait.Forever(func() {
|
||||
if backOff.IsInBackOffSinceUpdate(backOffID, time.Now()) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.doWatch(); err != nil {
|
||||
klog.ErrorS(err, "Unable to read config path", "path", s.path)
|
||||
if _, retryable := err.(*retryableError); !retryable {
|
||||
backOff.Next(backOffID, time.Now())
|
||||
}
|
||||
}
|
||||
}, retryPeriod)
|
||||
}
|
||||
|
||||
func (s *sourceFile) doWatch() error {
|
||||
_, err := os.Stat(s.path)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Emit an update with an empty PodList to allow FileSource to be marked as seen
|
||||
s.updates <- kubetypes.PodUpdate{Pods: []*v1.Pod{}, Op: kubetypes.SET, Source: kubetypes.FileSource}
|
||||
return &retryableError{"path does not exist, ignoring"}
|
||||
}
|
||||
|
||||
w, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create inotify: %v", err)
|
||||
}
|
||||
defer w.Close()
|
||||
|
||||
err = w.Add(s.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create inotify for path %q: %v", s.path, err)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case event := <-w.Events:
|
||||
if err = s.produceWatchEvent(&event); err != nil {
|
||||
return fmt.Errorf("error while processing inotify event (%+v): %v", event, err)
|
||||
}
|
||||
case err = <-w.Errors:
|
||||
return fmt.Errorf("error while watching %q: %v", s.path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sourceFile) produceWatchEvent(e *fsnotify.Event) error {
|
||||
// Ignore file start with dots
|
||||
if strings.HasPrefix(filepath.Base(e.Name), ".") {
|
||||
klog.V(4).InfoS("Ignored pod manifest, because it starts with dots", "eventName", e.Name)
|
||||
return nil
|
||||
}
|
||||
var eventType podEventType
|
||||
switch {
|
||||
case (e.Op & fsnotify.Create) > 0:
|
||||
eventType = podAdd
|
||||
case (e.Op & fsnotify.Write) > 0:
|
||||
eventType = podModify
|
||||
case (e.Op & fsnotify.Chmod) > 0:
|
||||
eventType = podModify
|
||||
case (e.Op & fsnotify.Remove) > 0:
|
||||
eventType = podDelete
|
||||
case (e.Op & fsnotify.Rename) > 0:
|
||||
eventType = podDelete
|
||||
default:
|
||||
// Ignore rest events
|
||||
return nil
|
||||
}
|
||||
|
||||
s.watchEvents <- &watchEvent{e.Name, eventType}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *sourceFile) consumeWatchEvent(e *watchEvent) error {
|
||||
switch e.eventType {
|
||||
case podAdd, podModify:
|
||||
pod, err := s.extractFromFile(e.fileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("can't process config file %q: %v", e.fileName, err)
|
||||
}
|
||||
return s.store.Add(pod)
|
||||
case podDelete:
|
||||
if objKey, keyExist := s.fileKeyMapping[e.fileName]; keyExist {
|
||||
pod, podExist, err := s.store.GetByKey(objKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !podExist {
|
||||
return fmt.Errorf("the pod with key %s doesn't exist in cache", objKey)
|
||||
}
|
||||
if err = s.store.Delete(pod); err != nil {
|
||||
return fmt.Errorf("failed to remove deleted pod from cache: %v", err)
|
||||
}
|
||||
delete(s.fileKeyMapping, e.fileName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
34
vendor/k8s.io/kubernetes/pkg/kubelet/config/file_unsupported.go
generated
vendored
Normal file
34
vendor/k8s.io/kubernetes/pkg/kubelet/config/file_unsupported.go
generated
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
func (s *sourceFile) startWatch() {
|
||||
klog.ErrorS(nil, "Watching source file is unsupported in this build")
|
||||
}
|
||||
|
||||
func (s *sourceFile) consumeWatchEvent(e *watchEvent) error {
|
||||
return fmt.Errorf("consuming watch event is unsupported in this build")
|
||||
}
|
59
vendor/k8s.io/kubernetes/pkg/kubelet/config/flags.go
generated
vendored
Normal file
59
vendor/k8s.io/kubernetes/pkg/kubelet/config/flags.go
generated
vendored
Normal file
@ -0,0 +1,59 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// ContainerRuntimeOptions defines options for the container runtime.
|
||||
type ContainerRuntimeOptions struct {
|
||||
// General Options.
|
||||
|
||||
// RuntimeCgroups that container runtime is expected to be isolated in.
|
||||
RuntimeCgroups string
|
||||
// PodSandboxImage is the image whose network/ipc namespaces
|
||||
// containers in each pod will use.
|
||||
PodSandboxImage string
|
||||
// Image credential provider plugin options
|
||||
|
||||
// ImageCredentialProviderConfigFile is the path to the credential provider plugin config file.
|
||||
// This config file is a specification for what credential providers are enabled and invoked
|
||||
// by the kubelet. The plugin config should contain information about what plugin binary
|
||||
// to execute and what container images the plugin should be called for.
|
||||
// +optional
|
||||
ImageCredentialProviderConfigFile string
|
||||
// ImageCredentialProviderBinDir is the path to the directory where credential provider plugin
|
||||
// binaries exist. The name of each plugin binary is expected to match the name of the plugin
|
||||
// specified in imageCredentialProviderConfigFile.
|
||||
// +optional
|
||||
ImageCredentialProviderBinDir string
|
||||
}
|
||||
|
||||
// AddFlags adds flags to the container runtime, according to ContainerRuntimeOptions.
|
||||
func (s *ContainerRuntimeOptions) AddFlags(fs *pflag.FlagSet) {
|
||||
// General settings.
|
||||
fs.StringVar(&s.RuntimeCgroups, "runtime-cgroups", s.RuntimeCgroups, "Optional absolute name of cgroups to create and run the runtime in.")
|
||||
fs.StringVar(&s.PodSandboxImage, "pod-infra-container-image", s.PodSandboxImage, fmt.Sprintf("Specified image will not be pruned by the image garbage collector. CRI implementations have their own configuration to set this image."))
|
||||
_ = fs.MarkDeprecated("pod-infra-container-image", "will be removed in 1.35. Image garbage collector will get sandbox image information from CRI.")
|
||||
|
||||
// Image credential provider settings.
|
||||
fs.StringVar(&s.ImageCredentialProviderConfigFile, "image-credential-provider-config", s.ImageCredentialProviderConfigFile, "The path to the credential provider plugin config file.")
|
||||
fs.StringVar(&s.ImageCredentialProviderBinDir, "image-credential-provider-bin-dir", s.ImageCredentialProviderBinDir, "The path to the directory where credential provider plugin binaries are located.")
|
||||
}
|
143
vendor/k8s.io/kubernetes/pkg/kubelet/config/http.go
generated
vendored
Normal file
143
vendor/k8s.io/kubernetes/pkg/kubelet/config/http.go
generated
vendored
Normal file
@ -0,0 +1,143 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
api "k8s.io/kubernetes/pkg/apis/core"
|
||||
kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
|
||||
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/klog/v2"
|
||||
utilio "k8s.io/utils/io"
|
||||
)
|
||||
|
||||
type sourceURL struct {
|
||||
url string
|
||||
header http.Header
|
||||
nodeName types.NodeName
|
||||
updates chan<- interface{}
|
||||
data []byte
|
||||
failureLogs int
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewSourceURL specifies the URL where to read the Pod configuration from, then watches it for changes.
|
||||
func NewSourceURL(url string, header http.Header, nodeName types.NodeName, period time.Duration, updates chan<- interface{}) {
|
||||
config := &sourceURL{
|
||||
url: url,
|
||||
header: header,
|
||||
nodeName: nodeName,
|
||||
updates: updates,
|
||||
data: nil,
|
||||
// Timing out requests leads to retries. This client is only used to
|
||||
// read the manifest URL passed to kubelet.
|
||||
client: &http.Client{Timeout: 10 * time.Second},
|
||||
}
|
||||
klog.V(1).InfoS("Watching URL", "URL", url)
|
||||
go wait.Until(config.run, period, wait.NeverStop)
|
||||
}
|
||||
|
||||
func (s *sourceURL) run() {
|
||||
if err := s.extractFromURL(); err != nil {
|
||||
// Don't log this multiple times per minute. The first few entries should be
|
||||
// enough to get the point across.
|
||||
if s.failureLogs < 3 {
|
||||
klog.InfoS("Failed to read pods from URL", "err", err)
|
||||
} else if s.failureLogs == 3 {
|
||||
klog.InfoS("Failed to read pods from URL. Dropping verbosity of this message to V(4)", "err", err)
|
||||
} else {
|
||||
klog.V(4).InfoS("Failed to read pods from URL", "err", err)
|
||||
}
|
||||
s.failureLogs++
|
||||
} else {
|
||||
if s.failureLogs > 0 {
|
||||
klog.InfoS("Successfully read pods from URL")
|
||||
s.failureLogs = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *sourceURL) applyDefaults(pod *api.Pod) error {
|
||||
return applyDefaults(pod, s.url, false, s.nodeName)
|
||||
}
|
||||
|
||||
func (s *sourceURL) extractFromURL() error {
|
||||
req, err := http.NewRequest("GET", s.url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header = s.header
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := utilio.ReadAtMost(resp.Body, maxConfigLength)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%v: %v", s.url, resp.Status)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
// Emit an update with an empty PodList to allow HTTPSource to be marked as seen
|
||||
s.updates <- kubetypes.PodUpdate{Pods: []*v1.Pod{}, Op: kubetypes.SET, Source: kubetypes.HTTPSource}
|
||||
return fmt.Errorf("zero-length data received from %v", s.url)
|
||||
}
|
||||
// Short circuit if the data has not changed since the last time it was read.
|
||||
if bytes.Equal(data, s.data) {
|
||||
return nil
|
||||
}
|
||||
s.data = data
|
||||
|
||||
// First try as it is a single pod.
|
||||
parsed, pod, singlePodErr := tryDecodeSinglePod(data, s.applyDefaults)
|
||||
if parsed {
|
||||
if singlePodErr != nil {
|
||||
// It parsed but could not be used.
|
||||
return singlePodErr
|
||||
}
|
||||
s.updates <- kubetypes.PodUpdate{Pods: []*v1.Pod{pod}, Op: kubetypes.SET, Source: kubetypes.HTTPSource}
|
||||
return nil
|
||||
}
|
||||
|
||||
// That didn't work, so try a list of pods.
|
||||
parsed, podList, multiPodErr := tryDecodePodList(data, s.applyDefaults)
|
||||
if parsed {
|
||||
if multiPodErr != nil {
|
||||
// It parsed but could not be used.
|
||||
return multiPodErr
|
||||
}
|
||||
pods := make([]*v1.Pod, 0, len(podList.Items))
|
||||
for i := range podList.Items {
|
||||
pods = append(pods, &podList.Items[i])
|
||||
}
|
||||
s.updates <- kubetypes.PodUpdate{Pods: pods, Op: kubetypes.SET, Source: kubetypes.HTTPSource}
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("%v: received '%v', but couldn't parse as "+
|
||||
"single (%v) or multiple pods (%v)",
|
||||
s.url, string(data), singlePodErr, multiPodErr)
|
||||
}
|
83
vendor/k8s.io/kubernetes/pkg/kubelet/config/mux.go
generated
vendored
Normal file
83
vendor/k8s.io/kubernetes/pkg/kubelet/config/mux.go
generated
vendored
Normal file
@ -0,0 +1,83 @@
|
||||
/*
|
||||
Copyright 2014 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package config
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/klog/v2"
|
||||
)
|
||||
|
||||
type merger interface {
|
||||
// Invoked when a change from a source is received. May also function as an incremental
|
||||
// merger if you wish to consume changes incrementally. Must be reentrant when more than
|
||||
// one source is defined.
|
||||
Merge(source string, update interface{}) error
|
||||
}
|
||||
|
||||
// mux is a class for merging configuration from multiple sources. Changes are
|
||||
// pushed via channels and sent to the merge function.
|
||||
type mux struct {
|
||||
// Invoked when an update is sent to a source.
|
||||
merger merger
|
||||
|
||||
// Sources and their lock.
|
||||
sourceLock sync.RWMutex
|
||||
// Maps source names to channels
|
||||
sources map[string]chan interface{}
|
||||
}
|
||||
|
||||
// newMux creates a new mux that can merge changes from multiple sources.
|
||||
func newMux(merger merger) *mux {
|
||||
mux := &mux{
|
||||
sources: make(map[string]chan interface{}),
|
||||
merger: merger,
|
||||
}
|
||||
return mux
|
||||
}
|
||||
|
||||
// ChannelWithContext returns a channel where a configuration source
|
||||
// can send updates of new configurations. Multiple calls with the same
|
||||
// source will return the same channel. This allows change and state based sources
|
||||
// to use the same channel. Different source names however will be treated as a
|
||||
// union.
|
||||
func (m *mux) ChannelWithContext(ctx context.Context, source string) chan interface{} {
|
||||
if len(source) == 0 {
|
||||
panic("Channel given an empty name")
|
||||
}
|
||||
m.sourceLock.Lock()
|
||||
defer m.sourceLock.Unlock()
|
||||
channel, exists := m.sources[source]
|
||||
if exists {
|
||||
return channel
|
||||
}
|
||||
newChannel := make(chan interface{})
|
||||
m.sources[source] = newChannel
|
||||
|
||||
go wait.Until(func() { m.listen(source, newChannel) }, 0, ctx.Done())
|
||||
return newChannel
|
||||
}
|
||||
|
||||
func (m *mux) listen(source string, listenChannel <-chan interface{}) {
|
||||
for update := range listenChannel {
|
||||
if err := m.merger.Merge(source, update); err != nil {
|
||||
klog.InfoS("failed merging update", "err", err)
|
||||
}
|
||||
}
|
||||
}
|
67
vendor/k8s.io/kubernetes/pkg/kubelet/config/sources.go
generated
vendored
Normal file
67
vendor/k8s.io/kubernetes/pkg/kubelet/config/sources.go
generated
vendored
Normal file
@ -0,0 +1,67 @@
|
||||
/*
|
||||
Copyright 2016 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package config implements the pod configuration readers.
|
||||
package config
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"k8s.io/apimachinery/pkg/util/sets"
|
||||
)
|
||||
|
||||
// SourcesReadyFn is function that returns true if the specified sources have been seen.
|
||||
type SourcesReadyFn func(sourcesSeen sets.Set[string]) bool
|
||||
|
||||
// SourcesReady tracks the set of configured sources seen by the kubelet.
|
||||
type SourcesReady interface {
|
||||
// AddSource adds the specified source to the set of sources managed.
|
||||
AddSource(source string)
|
||||
// AllReady returns true if the currently configured sources have all been seen.
|
||||
AllReady() bool
|
||||
}
|
||||
|
||||
// NewSourcesReady returns a SourcesReady with the specified function.
|
||||
func NewSourcesReady(sourcesReadyFn SourcesReadyFn) SourcesReady {
|
||||
return &sourcesImpl{
|
||||
sourcesSeen: sets.New[string](),
|
||||
sourcesReadyFn: sourcesReadyFn,
|
||||
}
|
||||
}
|
||||
|
||||
// sourcesImpl implements SourcesReady. It is thread-safe.
|
||||
type sourcesImpl struct {
|
||||
// lock protects access to sources seen.
|
||||
lock sync.RWMutex
|
||||
// set of sources seen.
|
||||
sourcesSeen sets.Set[string]
|
||||
// sourcesReady is a function that evaluates if the sources are ready.
|
||||
sourcesReadyFn SourcesReadyFn
|
||||
}
|
||||
|
||||
// Add adds the specified source to the set of sources managed.
|
||||
func (s *sourcesImpl) AddSource(source string) {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.sourcesSeen.Insert(source)
|
||||
}
|
||||
|
||||
// AllReady returns true if each configured source is ready.
|
||||
func (s *sourcesImpl) AllReady() bool {
|
||||
s.lock.RLock()
|
||||
defer s.lock.RUnlock()
|
||||
return s.sourcesReadyFn(s.sourcesSeen)
|
||||
}
|
Reference in New Issue
Block a user