vendor update for CSI 0.3.0

This commit is contained in:
gman
2018-07-18 16:47:22 +02:00
parent 6f484f92fc
commit 8ea659f0d5
6810 changed files with 438061 additions and 193861 deletions

View File

@ -0,0 +1,66 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
go_library(
name = "go_default_library",
srcs = [
"cache_based_manager.go",
"manager.go",
"watch_based_manager.go",
],
importpath = "k8s.io/kubernetes/pkg/kubelet/util/manager",
visibility = ["//visibility:public"],
deps = [
"//pkg/kubelet/util:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/etcd:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)
go_test(
name = "go_default_test",
srcs = [
"cache_based_manager_test.go",
"watch_based_manager_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/api/v1/pod:go_default_library",
"//pkg/apis/core/v1:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/testing:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@ -0,0 +1,272 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"fmt"
"strconv"
"sync"
"time"
"k8s.io/api/core/v1"
storageetcd "k8s.io/apiserver/pkg/storage/etcd"
"k8s.io/kubernetes/pkg/kubelet/util"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/sets"
)
// GetObjectTTLFunc defines a function to get value of TTL.
type GetObjectTTLFunc func() (time.Duration, bool)
// GetObjectFunc defines a function to get object with a given namespace and name.
type GetObjectFunc func(string, string, metav1.GetOptions) (runtime.Object, error)
type objectKey struct {
namespace string
name string
}
// objectStoreItems is a single item stored in objectStore.
type objectStoreItem struct {
refCount int
data *objectData
}
type objectData struct {
sync.Mutex
object runtime.Object
err error
lastUpdateTime time.Time
}
// objectStore is a local cache of objects.
type objectStore struct {
getObject GetObjectFunc
clock clock.Clock
lock sync.Mutex
items map[objectKey]*objectStoreItem
defaultTTL time.Duration
getTTL GetObjectTTLFunc
}
// NewObjectStore returns a new ttl-based instance of Store interface.
func NewObjectStore(getObject GetObjectFunc, clock clock.Clock, getTTL GetObjectTTLFunc, ttl time.Duration) Store {
return &objectStore{
getObject: getObject,
clock: clock,
items: make(map[objectKey]*objectStoreItem),
defaultTTL: ttl,
getTTL: getTTL,
}
}
func isObjectOlder(newObject, oldObject runtime.Object) bool {
if newObject == nil || oldObject == nil {
return false
}
newVersion, _ := storageetcd.Versioner.ObjectResourceVersion(newObject)
oldVersion, _ := storageetcd.Versioner.ObjectResourceVersion(oldObject)
return newVersion < oldVersion
}
func (s *objectStore) AddReference(namespace, name string) {
key := objectKey{namespace: namespace, name: name}
// AddReference is called from RegisterPod, thus it needs to be efficient.
// Thus Add() is only increasing refCount and generation of a given object.
// Then Get() is responsible for fetching if needed.
s.lock.Lock()
defer s.lock.Unlock()
item, exists := s.items[key]
if !exists {
item = &objectStoreItem{
refCount: 0,
data: &objectData{},
}
s.items[key] = item
}
item.refCount++
// This will trigger fetch on the next Get() operation.
item.data = nil
}
func (s *objectStore) DeleteReference(namespace, name string) {
key := objectKey{namespace: namespace, name: name}
s.lock.Lock()
defer s.lock.Unlock()
if item, ok := s.items[key]; ok {
item.refCount--
if item.refCount == 0 {
delete(s.items, key)
}
}
}
// GetObjectTTLFromNodeFunc returns a function that returns TTL value
// from a given Node object.
func GetObjectTTLFromNodeFunc(getNode func() (*v1.Node, error)) GetObjectTTLFunc {
return func() (time.Duration, bool) {
node, err := getNode()
if err != nil {
return time.Duration(0), false
}
if node != nil && node.Annotations != nil {
if value, ok := node.Annotations[v1.ObjectTTLAnnotationKey]; ok {
if intValue, err := strconv.Atoi(value); err == nil {
return time.Duration(intValue) * time.Second, true
}
}
}
return time.Duration(0), false
}
}
func (s *objectStore) isObjectFresh(data *objectData) bool {
objectTTL := s.defaultTTL
if ttl, ok := s.getTTL(); ok {
objectTTL = ttl
}
return s.clock.Now().Before(data.lastUpdateTime.Add(objectTTL))
}
func (s *objectStore) Get(namespace, name string) (runtime.Object, error) {
key := objectKey{namespace: namespace, name: name}
data := func() *objectData {
s.lock.Lock()
defer s.lock.Unlock()
item, exists := s.items[key]
if !exists {
return nil
}
if item.data == nil {
item.data = &objectData{}
}
return item.data
}()
if data == nil {
return nil, fmt.Errorf("object %q/%q not registered", namespace, name)
}
// After updating data in objectStore, lock the data, fetch object if
// needed and return data.
data.Lock()
defer data.Unlock()
if data.err != nil || !s.isObjectFresh(data) {
opts := metav1.GetOptions{}
if data.object != nil && data.err == nil {
// This is just a periodic refresh of an object we successfully fetched previously.
// In this case, server data from apiserver cache to reduce the load on both
// etcd and apiserver (the cache is eventually consistent).
util.FromApiserverCache(&opts)
}
object, err := s.getObject(namespace, name, opts)
if err != nil && !apierrors.IsNotFound(err) && data.object == nil && data.err == nil {
// Couldn't fetch the latest object, but there is no cached data to return.
// Return the fetch result instead.
return object, err
}
if (err == nil && !isObjectOlder(object, data.object)) || apierrors.IsNotFound(err) {
// If the fetch succeeded with a newer version of the object, or if the
// object could not be found in the apiserver, update the cached data to
// reflect the current status.
data.object = object
data.err = err
data.lastUpdateTime = s.clock.Now()
}
}
return data.object, data.err
}
// cacheBasedManager keeps a store with objects necessary
// for registered pods. Different implementations of the store
// may result in different semantics for freshness of objects
// (e.g. ttl-based implementation vs watch-based implementation).
type cacheBasedManager struct {
objectStore Store
getReferencedObjects func(*v1.Pod) sets.String
lock sync.Mutex
registeredPods map[objectKey]*v1.Pod
}
func (c *cacheBasedManager) GetObject(namespace, name string) (runtime.Object, error) {
return c.objectStore.Get(namespace, name)
}
func (c *cacheBasedManager) RegisterPod(pod *v1.Pod) {
names := c.getReferencedObjects(pod)
c.lock.Lock()
defer c.lock.Unlock()
for name := range names {
c.objectStore.AddReference(pod.Namespace, name)
}
var prev *v1.Pod
key := objectKey{namespace: pod.Namespace, name: pod.Name}
prev = c.registeredPods[key]
c.registeredPods[key] = pod
if prev != nil {
for name := range c.getReferencedObjects(prev) {
// On an update, the .Add() call above will have re-incremented the
// ref count of any existing object, so any objects that are in both
// names and prev need to have their ref counts decremented. Any that
// are only in prev need to be completely removed. This unconditional
// call takes care of both cases.
c.objectStore.DeleteReference(prev.Namespace, name)
}
}
}
func (c *cacheBasedManager) UnregisterPod(pod *v1.Pod) {
var prev *v1.Pod
key := objectKey{namespace: pod.Namespace, name: pod.Name}
c.lock.Lock()
defer c.lock.Unlock()
prev = c.registeredPods[key]
delete(c.registeredPods, key)
if prev != nil {
for name := range c.getReferencedObjects(prev) {
c.objectStore.DeleteReference(prev.Namespace, name)
}
}
}
// NewCacheBasedManager creates a manager that keeps a cache of all objects
// necessary for registered pods.
// It implements the following logic:
// - whenever a pod is created or updated, the cached versions of all objects
// is is referencing are invalidated
// - every GetObject() call tries to fetch the value from local cache; if it is
// not there, invalidated or too old, we fetch it from apiserver and refresh the
// value in cache; otherwise it is just fetched from cache
func NewCacheBasedManager(objectStore Store, getReferencedObjects func(*v1.Pod) sets.String) Manager {
return &cacheBasedManager{
objectStore: objectStore,
getReferencedObjects: getReferencedObjects,
registeredPods: make(map[objectKey]*v1.Pod),
}
}

View File

@ -0,0 +1,563 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"fmt"
"reflect"
"strings"
"sync"
"testing"
"time"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/sets"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
"github.com/stretchr/testify/assert"
)
func checkObject(t *testing.T, store *objectStore, ns, name string, shouldExist bool) {
_, err := store.Get(ns, name)
if shouldExist && err != nil {
t.Errorf("unexpected actions: %#v", err)
}
if !shouldExist && (err == nil || !strings.Contains(err.Error(), fmt.Sprintf("object %q/%q not registered", ns, name))) {
t.Errorf("unexpected actions: %#v", err)
}
}
func noObjectTTL() (time.Duration, bool) {
return time.Duration(0), false
}
func getSecret(fakeClient clientset.Interface) GetObjectFunc {
return func(namespace, name string, opts metav1.GetOptions) (runtime.Object, error) {
return fakeClient.CoreV1().Secrets(namespace).Get(name, opts)
}
}
func newSecretStore(fakeClient clientset.Interface, clock clock.Clock, getTTL GetObjectTTLFunc, ttl time.Duration) *objectStore {
return &objectStore{
getObject: getSecret(fakeClient),
clock: clock,
items: make(map[objectKey]*objectStoreItem),
defaultTTL: ttl,
getTTL: getTTL,
}
}
func getSecretNames(pod *v1.Pod) sets.String {
result := sets.NewString()
podutil.VisitPodSecretNames(pod, func(name string) bool {
result.Insert(name)
return true
})
return result
}
func newCacheBasedSecretManager(store Store) Manager {
return NewCacheBasedManager(store, getSecretNames)
}
func TestSecretStore(t *testing.T) {
fakeClient := &fake.Clientset{}
store := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0)
store.AddReference("ns1", "name1")
store.AddReference("ns2", "name2")
store.AddReference("ns1", "name1")
store.AddReference("ns1", "name1")
store.DeleteReference("ns1", "name1")
store.DeleteReference("ns2", "name2")
store.AddReference("ns3", "name3")
// Adds don't issue Get requests.
actions := fakeClient.Actions()
assert.Equal(t, 0, len(actions), "unexpected actions: %#v", actions)
// Should issue Get request
store.Get("ns1", "name1")
// Shouldn't issue Get request, as secret is not registered
store.Get("ns2", "name2")
// Should issue Get request
store.Get("ns3", "name3")
actions = fakeClient.Actions()
assert.Equal(t, 2, len(actions), "unexpected actions: %#v", actions)
for _, a := range actions {
assert.True(t, a.Matches("get", "secrets"), "unexpected actions: %#v", a)
}
checkObject(t, store, "ns1", "name1", true)
checkObject(t, store, "ns2", "name2", false)
checkObject(t, store, "ns3", "name3", true)
checkObject(t, store, "ns4", "name4", false)
}
func TestSecretStoreDeletingSecret(t *testing.T) {
fakeClient := &fake.Clientset{}
store := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0)
store.AddReference("ns", "name")
result := &v1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: "name", ResourceVersion: "10"}}
fakeClient.AddReactor("get", "secrets", func(action core.Action) (bool, runtime.Object, error) {
return true, result, nil
})
secret, err := store.Get("ns", "name")
if err != nil {
t.Errorf("Unexpected error: %v", err)
}
if !reflect.DeepEqual(secret, result) {
t.Errorf("Unexpected secret: %v", secret)
}
fakeClient.PrependReactor("get", "secrets", func(action core.Action) (bool, runtime.Object, error) {
return true, &v1.Secret{}, apierrors.NewNotFound(v1.Resource("secret"), "name")
})
secret, err = store.Get("ns", "name")
if err == nil || !apierrors.IsNotFound(err) {
t.Errorf("Unexpected error: %v", err)
}
if !reflect.DeepEqual(secret, &v1.Secret{}) {
t.Errorf("Unexpected secret: %v", secret)
}
}
func TestSecretStoreGetAlwaysRefresh(t *testing.T) {
fakeClient := &fake.Clientset{}
fakeClock := clock.NewFakeClock(time.Now())
store := newSecretStore(fakeClient, fakeClock, noObjectTTL, 0)
for i := 0; i < 10; i++ {
store.AddReference(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i))
}
fakeClient.ClearActions()
wg := sync.WaitGroup{}
wg.Add(100)
for i := 0; i < 100; i++ {
go func(i int) {
store.Get(fmt.Sprintf("ns-%d", i%10), fmt.Sprintf("name-%d", i%10))
wg.Done()
}(i)
}
wg.Wait()
actions := fakeClient.Actions()
assert.Equal(t, 100, len(actions), "unexpected actions: %#v", actions)
for _, a := range actions {
assert.True(t, a.Matches("get", "secrets"), "unexpected actions: %#v", a)
}
}
func TestSecretStoreGetNeverRefresh(t *testing.T) {
fakeClient := &fake.Clientset{}
fakeClock := clock.NewFakeClock(time.Now())
store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute)
for i := 0; i < 10; i++ {
store.AddReference(fmt.Sprintf("ns-%d", i), fmt.Sprintf("name-%d", i))
}
fakeClient.ClearActions()
wg := sync.WaitGroup{}
wg.Add(100)
for i := 0; i < 100; i++ {
go func(i int) {
store.Get(fmt.Sprintf("ns-%d", i%10), fmt.Sprintf("name-%d", i%10))
wg.Done()
}(i)
}
wg.Wait()
actions := fakeClient.Actions()
// Only first Get, should forward the Get request.
assert.Equal(t, 10, len(actions), "unexpected actions: %#v", actions)
}
func TestCustomTTL(t *testing.T) {
ttl := time.Duration(0)
ttlExists := false
customTTL := func() (time.Duration, bool) {
return ttl, ttlExists
}
fakeClient := &fake.Clientset{}
fakeClock := clock.NewFakeClock(time.Time{})
store := newSecretStore(fakeClient, fakeClock, customTTL, time.Minute)
store.AddReference("ns", "name")
store.Get("ns", "name")
fakeClient.ClearActions()
// Set 0-ttl and see if that works.
ttl = time.Duration(0)
ttlExists = true
store.Get("ns", "name")
actions := fakeClient.Actions()
assert.Equal(t, 1, len(actions), "unexpected actions: %#v", actions)
fakeClient.ClearActions()
// Set 5-minute ttl and see if this works.
ttl = time.Duration(5) * time.Minute
store.Get("ns", "name")
actions = fakeClient.Actions()
assert.Equal(t, 0, len(actions), "unexpected actions: %#v", actions)
// Still no effect after 4 minutes.
fakeClock.Step(4 * time.Minute)
store.Get("ns", "name")
actions = fakeClient.Actions()
assert.Equal(t, 0, len(actions), "unexpected actions: %#v", actions)
// Now it should have an effect.
fakeClock.Step(time.Minute)
store.Get("ns", "name")
actions = fakeClient.Actions()
assert.Equal(t, 1, len(actions), "unexpected actions: %#v", actions)
fakeClient.ClearActions()
// Now remove the custom ttl and see if that works.
ttlExists = false
fakeClock.Step(55 * time.Second)
store.Get("ns", "name")
actions = fakeClient.Actions()
assert.Equal(t, 0, len(actions), "unexpected actions: %#v", actions)
// Pass the minute and it should be triggered now.
fakeClock.Step(5 * time.Second)
store.Get("ns", "name")
actions = fakeClient.Actions()
assert.Equal(t, 1, len(actions), "unexpected actions: %#v", actions)
}
func TestParseNodeAnnotation(t *testing.T) {
testCases := []struct {
node *v1.Node
err error
exists bool
ttl time.Duration
}{
{
node: nil,
err: fmt.Errorf("error"),
exists: false,
},
{
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node",
},
},
exists: false,
},
{
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node",
Annotations: map[string]string{},
},
},
exists: false,
},
{
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node",
Annotations: map[string]string{v1.ObjectTTLAnnotationKey: "bad"},
},
},
exists: false,
},
{
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node",
Annotations: map[string]string{v1.ObjectTTLAnnotationKey: "0"},
},
},
exists: true,
ttl: time.Duration(0),
},
{
node: &v1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "node",
Annotations: map[string]string{v1.ObjectTTLAnnotationKey: "60"},
},
},
exists: true,
ttl: time.Minute,
},
}
for i, testCase := range testCases {
getNode := func() (*v1.Node, error) { return testCase.node, testCase.err }
ttl, exists := GetObjectTTLFromNodeFunc(getNode)()
if exists != testCase.exists {
t.Errorf("%d: incorrect parsing: %t", i, exists)
continue
}
if exists && ttl != testCase.ttl {
t.Errorf("%d: incorrect ttl: %v", i, ttl)
}
}
}
type envSecrets struct {
envVarNames []string
envFromNames []string
}
type secretsToAttach struct {
imagePullSecretNames []string
containerEnvSecrets []envSecrets
}
func podWithSecrets(ns, podName string, toAttach secretsToAttach) *v1.Pod {
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
Namespace: ns,
Name: podName,
},
Spec: v1.PodSpec{},
}
for _, name := range toAttach.imagePullSecretNames {
pod.Spec.ImagePullSecrets = append(
pod.Spec.ImagePullSecrets, v1.LocalObjectReference{Name: name})
}
for i, secrets := range toAttach.containerEnvSecrets {
container := v1.Container{
Name: fmt.Sprintf("container-%d", i),
}
for _, name := range secrets.envFromNames {
envFrom := v1.EnvFromSource{
SecretRef: &v1.SecretEnvSource{
LocalObjectReference: v1.LocalObjectReference{
Name: name,
},
},
}
container.EnvFrom = append(container.EnvFrom, envFrom)
}
for _, name := range secrets.envVarNames {
envSource := &v1.EnvVarSource{
SecretKeyRef: &v1.SecretKeySelector{
LocalObjectReference: v1.LocalObjectReference{
Name: name,
},
},
}
container.Env = append(container.Env, v1.EnvVar{ValueFrom: envSource})
}
pod.Spec.Containers = append(pod.Spec.Containers, container)
}
return pod
}
func TestCacheInvalidation(t *testing.T) {
fakeClient := &fake.Clientset{}
fakeClock := clock.NewFakeClock(time.Now())
store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute)
manager := newCacheBasedSecretManager(store)
// Create a pod with some secrets.
s1 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s1"}, envFromNames: []string{"s10"}},
{envVarNames: []string{"s2"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name1", s1))
// Fetch both secrets - this should triggger get operations.
store.Get("ns1", "s1")
store.Get("ns1", "s10")
store.Get("ns1", "s2")
actions := fakeClient.Actions()
assert.Equal(t, 3, len(actions), "unexpected actions: %#v", actions)
fakeClient.ClearActions()
// Update a pod with a new secret.
s2 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s1"}},
{envVarNames: []string{"s2"}, envFromNames: []string{"s20"}},
{envVarNames: []string{"s3"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name1", s2))
// All secrets should be invalidated - this should trigger get operations.
store.Get("ns1", "s1")
store.Get("ns1", "s2")
store.Get("ns1", "s20")
store.Get("ns1", "s3")
actions = fakeClient.Actions()
assert.Equal(t, 4, len(actions), "unexpected actions: %#v", actions)
fakeClient.ClearActions()
// Create a new pod that is refencing the first three secrets - those should
// be invalidated.
manager.RegisterPod(podWithSecrets("ns1", "name2", s1))
store.Get("ns1", "s1")
store.Get("ns1", "s10")
store.Get("ns1", "s2")
store.Get("ns1", "s20")
store.Get("ns1", "s3")
actions = fakeClient.Actions()
assert.Equal(t, 3, len(actions), "unexpected actions: %#v", actions)
fakeClient.ClearActions()
}
func TestCacheRefcounts(t *testing.T) {
fakeClient := &fake.Clientset{}
fakeClock := clock.NewFakeClock(time.Now())
store := newSecretStore(fakeClient, fakeClock, noObjectTTL, time.Minute)
manager := newCacheBasedSecretManager(store)
s1 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s1"}, envFromNames: []string{"s10"}},
{envVarNames: []string{"s2"}},
{envVarNames: []string{"s3"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name1", s1))
manager.RegisterPod(podWithSecrets("ns1", "name2", s1))
s2 := secretsToAttach{
imagePullSecretNames: []string{"s2"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s4"}},
{envVarNames: []string{"s5"}, envFromNames: []string{"s50"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name2", s2))
manager.RegisterPod(podWithSecrets("ns1", "name3", s2))
manager.RegisterPod(podWithSecrets("ns1", "name4", s2))
manager.UnregisterPod(podWithSecrets("ns1", "name3", s2))
s3 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s3"}, envFromNames: []string{"s30"}},
{envVarNames: []string{"s5"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name5", s3))
manager.RegisterPod(podWithSecrets("ns1", "name6", s3))
s4 := secretsToAttach{
imagePullSecretNames: []string{"s3"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s6"}},
{envFromNames: []string{"s60"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name7", s4))
manager.UnregisterPod(podWithSecrets("ns1", "name7", s4))
// Also check the Add + Update + Remove scenario.
manager.RegisterPod(podWithSecrets("ns1", "other-name", s1))
manager.RegisterPod(podWithSecrets("ns1", "other-name", s2))
manager.UnregisterPod(podWithSecrets("ns1", "other-name", s2))
s5 := secretsToAttach{
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s7"}},
{envFromNames: []string{"s70"}},
},
}
// Check the no-op update scenario
manager.RegisterPod(podWithSecrets("ns1", "noop-pod", s5))
manager.RegisterPod(podWithSecrets("ns1", "noop-pod", s5))
// Now we have: 3 pods with s1, 2 pods with s2 and 2 pods with s3, 0 pods with s4.
refs := func(ns, name string) int {
store.lock.Lock()
defer store.lock.Unlock()
item, ok := store.items[objectKey{ns, name}]
if !ok {
return 0
}
return item.refCount
}
assert.Equal(t, 3, refs("ns1", "s1"))
assert.Equal(t, 1, refs("ns1", "s10"))
assert.Equal(t, 3, refs("ns1", "s2"))
assert.Equal(t, 3, refs("ns1", "s3"))
assert.Equal(t, 2, refs("ns1", "s30"))
assert.Equal(t, 2, refs("ns1", "s4"))
assert.Equal(t, 4, refs("ns1", "s5"))
assert.Equal(t, 2, refs("ns1", "s50"))
assert.Equal(t, 0, refs("ns1", "s6"))
assert.Equal(t, 0, refs("ns1", "s60"))
assert.Equal(t, 1, refs("ns1", "s7"))
assert.Equal(t, 1, refs("ns1", "s70"))
}
func TestCacheBasedSecretManager(t *testing.T) {
fakeClient := &fake.Clientset{}
store := newSecretStore(fakeClient, clock.RealClock{}, noObjectTTL, 0)
manager := newCacheBasedSecretManager(store)
// Create a pod with some secrets.
s1 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s1"}},
{envVarNames: []string{"s2"}},
{envFromNames: []string{"s20"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name1", s1))
// Update the pod with a different secrets.
s2 := secretsToAttach{
imagePullSecretNames: []string{"s1"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s3"}},
{envVarNames: []string{"s4"}},
{envFromNames: []string{"s40"}},
},
}
manager.RegisterPod(podWithSecrets("ns1", "name1", s2))
// Create another pod, but with same secrets in different namespace.
manager.RegisterPod(podWithSecrets("ns2", "name2", s2))
// Create and delete a pod with some other secrets.
s3 := secretsToAttach{
imagePullSecretNames: []string{"s5"},
containerEnvSecrets: []envSecrets{
{envVarNames: []string{"s6"}},
{envFromNames: []string{"s60"}},
},
}
manager.RegisterPod(podWithSecrets("ns3", "name", s3))
manager.UnregisterPod(podWithSecrets("ns3", "name", s3))
// We should have only: s1, s3 and s4 secrets in namespaces: ns1 and ns2.
for _, ns := range []string{"ns1", "ns2", "ns3"} {
for _, secret := range []string{"s1", "s2", "s3", "s4", "s5", "s6", "s20", "s40", "s50"} {
shouldExist :=
(secret == "s1" || secret == "s3" || secret == "s4" || secret == "s40") && (ns == "ns1" || ns == "ns2")
checkObject(t, store, ns, secret, shouldExist)
}
}
}

View File

@ -0,0 +1,56 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// Manager is the interface for registering and unregistering
// objects referenced by pods in the underlying cache and
// extracting those from that cache if needed.
type Manager interface {
// Get object by its namespace and name.
GetObject(namespace, name string) (runtime.Object, error)
// WARNING: Register/UnregisterPod functions should be efficient,
// i.e. should not block on network operations.
// RegisterPod registers all objects referenced from a given pod.
RegisterPod(pod *v1.Pod)
// UnregisterPod unregisters objects referenced from a given pod that are not
// used by any other registered pod.
UnregisterPod(pod *v1.Pod)
}
// Store is the interface for a object cache that
// can be used by cacheBasedManager.
type Store interface {
// AddReference adds a reference to the object to the store.
// Note that multiple additions to the store has to be allowed
// in the implementations and effectively treated as refcounted.
AddReference(namespace, name string)
// DeleteReference deletes reference to the object from the store.
// Note that object should be deleted only when there was a
// corresponding Delete call for each of Add calls (effectively
// when refcount was reduced to zero).
DeleteReference(namespace, name string)
// Get an object from a store.
Get(namespace, name string) (runtime.Object, error)
}

View File

@ -0,0 +1,194 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// TODO: We did some scalability tests and using watchBasedManager
// seems to help with apiserver performance at scale visibly.
// No issues we also observed at the scale of ~200k watchers with a
// single apiserver.
// However, we need to perform more extensive testing before we
// enable this in production setups.
package manager
import (
"fmt"
"sync"
"time"
"k8s.io/api/core/v1"
"k8s.io/client-go/tools/cache"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
)
type listObjectFunc func(string, metav1.ListOptions) (runtime.Object, error)
type watchObjectFunc func(string, metav1.ListOptions) (watch.Interface, error)
type newObjectFunc func() runtime.Object
// objectCacheItem is a single item stored in objectCache.
type objectCacheItem struct {
refCount int
store cache.Store
hasSynced func() (bool, error)
stopCh chan struct{}
}
// objectCache is a local cache of objects propagated via
// individual watches.
type objectCache struct {
listObject listObjectFunc
watchObject watchObjectFunc
newObject newObjectFunc
groupResource schema.GroupResource
lock sync.Mutex
items map[objectKey]*objectCacheItem
}
// NewObjectCache returns a new watch-based instance of Store interface.
func NewObjectCache(listObject listObjectFunc, watchObject watchObjectFunc, newObject newObjectFunc, groupResource schema.GroupResource) Store {
return &objectCache{
listObject: listObject,
watchObject: watchObject,
newObject: newObject,
groupResource: groupResource,
items: make(map[objectKey]*objectCacheItem),
}
}
func (c *objectCache) newStore() cache.Store {
// TODO: We may consider created a dedicated store keeping just a single
// item, instead of using a generic store implementation for this purpose.
// However, simple benchmarks show that memory overhead in that case is
// decrease from ~600B to ~300B per object. So we are not optimizing it
// until we will see a good reason for that.
return cache.NewStore(cache.MetaNamespaceKeyFunc)
}
func (c *objectCache) newReflector(namespace, name string) *objectCacheItem {
fieldSelector := fields.Set{"metadata.name": name}.AsSelector().String()
listFunc := func(options metav1.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return c.listObject(namespace, options)
}
watchFunc := func(options metav1.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return c.watchObject(namespace, options)
}
store := c.newStore()
reflector := cache.NewNamedReflector(
fmt.Sprintf("object-%q/%q", namespace, name),
&cache.ListWatch{ListFunc: listFunc, WatchFunc: watchFunc},
c.newObject(),
store,
0,
)
stopCh := make(chan struct{})
go reflector.Run(stopCh)
return &objectCacheItem{
refCount: 0,
store: store,
hasSynced: func() (bool, error) { return reflector.LastSyncResourceVersion() != "", nil },
stopCh: stopCh,
}
}
func (c *objectCache) AddReference(namespace, name string) {
key := objectKey{namespace: namespace, name: name}
// AddReference is called from RegisterPod thus it needs to be efficient.
// Thus, it is only increaisng refCount and in case of first registration
// of a given object it starts corresponding reflector.
// It's responsibility of the first Get operation to wait until the
// reflector propagated the store.
c.lock.Lock()
defer c.lock.Unlock()
item, exists := c.items[key]
if !exists {
item = c.newReflector(namespace, name)
c.items[key] = item
}
item.refCount++
}
func (c *objectCache) DeleteReference(namespace, name string) {
key := objectKey{namespace: namespace, name: name}
c.lock.Lock()
defer c.lock.Unlock()
if item, ok := c.items[key]; ok {
item.refCount--
if item.refCount == 0 {
// Stop the underlying reflector.
close(item.stopCh)
delete(c.items, key)
}
}
}
// key returns key of an object with a given name and namespace.
// This has to be in-sync with cache.MetaNamespaceKeyFunc.
func (c *objectCache) key(namespace, name string) string {
if len(namespace) > 0 {
return namespace + "/" + name
}
return name
}
func (c *objectCache) Get(namespace, name string) (runtime.Object, error) {
key := objectKey{namespace: namespace, name: name}
c.lock.Lock()
item, exists := c.items[key]
c.lock.Unlock()
if !exists {
return nil, fmt.Errorf("object %q/%q not registered", namespace, name)
}
if err := wait.PollImmediate(10*time.Millisecond, time.Second, item.hasSynced); err != nil {
return nil, fmt.Errorf("couldn't propagate object cache: %v", err)
}
obj, exists, err := item.store.GetByKey(c.key(namespace, name))
if err != nil {
return nil, err
}
if !exists {
return nil, apierrors.NewNotFound(c.groupResource, name)
}
if object, ok := obj.(runtime.Object); ok {
return object, nil
}
return nil, fmt.Errorf("unexpected object type: %v", obj)
}
// NewWatchBasedManager creates a manager that keeps a cache of all objects
// necessary for registered pods.
// It implements the following logic:
// - whenever a pod is created or updated, we start individual watches for all
// referenced objects that aren't referenced from other registered pods
// - every GetObject() returns a value from local cache propagated via watches
func NewWatchBasedManager(listObject listObjectFunc, watchObject watchObjectFunc, newObject newObjectFunc, groupResource schema.GroupResource, getReferencedObjects func(*v1.Pod) sets.String) Manager {
objectStore := NewObjectCache(listObject, watchObject, newObject, groupResource)
return NewCacheBasedManager(objectStore, getReferencedObjects)
}

View File

@ -0,0 +1,184 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package manager
import (
"fmt"
"strings"
"testing"
"time"
"k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
corev1 "k8s.io/kubernetes/pkg/apis/core/v1"
"github.com/stretchr/testify/assert"
)
func listSecret(fakeClient clientset.Interface) listObjectFunc {
return func(namespace string, opts metav1.ListOptions) (runtime.Object, error) {
return fakeClient.CoreV1().Secrets(namespace).List(opts)
}
}
func watchSecret(fakeClient clientset.Interface) watchObjectFunc {
return func(namespace string, opts metav1.ListOptions) (watch.Interface, error) {
return fakeClient.CoreV1().Secrets(namespace).Watch(opts)
}
}
func newSecretCache(fakeClient clientset.Interface) *objectCache {
return &objectCache{
listObject: listSecret(fakeClient),
watchObject: watchSecret(fakeClient),
newObject: func() runtime.Object { return &v1.Secret{} },
groupResource: corev1.Resource("secret"),
items: make(map[objectKey]*objectCacheItem),
}
}
func TestSecretCache(t *testing.T) {
fakeClient := &fake.Clientset{}
listReactor := func(a core.Action) (bool, runtime.Object, error) {
result := &v1.SecretList{
ListMeta: metav1.ListMeta{
ResourceVersion: "123",
},
}
return true, result, nil
}
fakeClient.AddReactor("list", "secrets", listReactor)
fakeWatch := watch.NewFake()
fakeClient.AddWatchReactor("secrets", core.DefaultWatchReactor(fakeWatch, nil))
store := newSecretCache(fakeClient)
store.AddReference("ns", "name")
_, err := store.Get("ns", "name")
if !apierrors.IsNotFound(err) {
t.Errorf("Expected NotFound error, got: %v", err)
}
// Eventually we should be able to read added secret.
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "name", Namespace: "ns", ResourceVersion: "125"},
}
fakeWatch.Add(secret)
getFn := func() (bool, error) {
object, err := store.Get("ns", "name")
if err != nil {
if apierrors.IsNotFound(err) {
return false, nil
}
return false, err
}
secret := object.(*v1.Secret)
if secret == nil || secret.Name != "name" || secret.Namespace != "ns" {
return false, fmt.Errorf("unexpected secret: %v", secret)
}
return true, nil
}
if err := wait.PollImmediate(10*time.Millisecond, time.Second, getFn); err != nil {
t.Errorf("unexpected error: %v", err)
}
// Eventually we should observer secret deletion.
fakeWatch.Delete(secret)
getFn = func() (bool, error) {
_, err := store.Get("ns", "name")
if err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, err
}
return false, nil
}
if err := wait.PollImmediate(10*time.Millisecond, time.Second, getFn); err != nil {
t.Errorf("unexpected error: %v", err)
}
store.DeleteReference("ns", "name")
_, err = store.Get("ns", "name")
if err == nil || !strings.Contains(err.Error(), "not registered") {
t.Errorf("unexpected error: %v", err)
}
}
func TestSecretCacheMultipleRegistrations(t *testing.T) {
fakeClient := &fake.Clientset{}
listReactor := func(a core.Action) (bool, runtime.Object, error) {
result := &v1.SecretList{
ListMeta: metav1.ListMeta{
ResourceVersion: "123",
},
}
return true, result, nil
}
fakeClient.AddReactor("list", "secrets", listReactor)
fakeWatch := watch.NewFake()
fakeClient.AddWatchReactor("secrets", core.DefaultWatchReactor(fakeWatch, nil))
store := newSecretCache(fakeClient)
store.AddReference("ns", "name")
// This should trigger List and Watch actions eventually.
actionsFn := func() (bool, error) {
actions := fakeClient.Actions()
if len(actions) > 2 {
return false, fmt.Errorf("too many actions: %v", actions)
}
if len(actions) < 2 {
return false, nil
}
if actions[0].GetVerb() != "list" || actions[1].GetVerb() != "watch" {
return false, fmt.Errorf("unexpected actions: %v", actions)
}
return true, nil
}
if err := wait.PollImmediate(10*time.Millisecond, time.Second, actionsFn); err != nil {
t.Errorf("unexpected error: %v", err)
}
// Next registrations shouldn't trigger any new actions.
for i := 0; i < 20; i++ {
store.AddReference("ns", "name")
store.DeleteReference("ns", "name")
}
actions := fakeClient.Actions()
assert.Equal(t, 2, len(actions), "unexpected actions: %#v", actions)
// Final delete also doesn't trigger any action.
store.DeleteReference("ns", "name")
_, err := store.Get("ns", "name")
if err == nil || !strings.Contains(err.Error(), "not registered") {
t.Errorf("unexpected error: %v", err)
}
actions = fakeClient.Actions()
assert.Equal(t, 2, len(actions), "unexpected actions: %#v", actions)
}