vendor update for CSI 0.3.0

This commit is contained in:
gman
2018-07-18 16:47:22 +02:00
parent 6f484f92fc
commit 8ea659f0d5
6810 changed files with 438061 additions and 193861 deletions

View File

@ -9,48 +9,44 @@ load(
go_test(
name = "go_default_test",
srcs = [
"checkpoint_test.go",
"configmap_test.go",
"download_test.go",
],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//pkg/kubelet/kubeletconfig/util/test:go_default_library",
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)
go_library(
name = "go_default_library",
srcs = [
"checkpoint.go",
"configmap.go",
"download.go",
],
importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint",
deps = [
"//pkg/api/legacyscheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/kubeletconfig/status:go_default_library",
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//pkg/kubelet/kubeletconfig/util/log:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
],
)

View File

@ -1,72 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package checkpoint
import (
"fmt"
apiv1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
)
// Checkpoint represents a local copy of a config source (payload) object
type Checkpoint interface {
// UID returns the UID of the config source object behind the Checkpoint
UID() string
// Parse extracts the KubeletConfiguration from the checkpoint, applies defaults, and converts to the internal type
Parse() (*kubeletconfig.KubeletConfiguration, error)
// Encode returns a []byte representation of the config source object behind the Checkpoint
Encode() ([]byte, error)
// object returns the underlying checkpointed object. If you want to compare sources for equality, use EqualCheckpoints,
// which compares the underlying checkpointed objects for semantic API equality.
object() interface{}
}
// DecodeCheckpoint is a helper for using the apimachinery to decode serialized checkpoints
func DecodeCheckpoint(data []byte) (Checkpoint, error) {
// decode the checkpoint
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data)
if err != nil {
return nil, fmt.Errorf("failed to decode, error: %v", err)
}
// TODO(mtaufen): for now we assume we are trying to load a ConfigMap checkpoint, may need to extend this if we allow other checkpoint types
// convert it to the external ConfigMap type, so we're consistently working with the external type outside of the on-disk representation
cm := &apiv1.ConfigMap{}
err = legacyscheme.Scheme.Convert(obj, cm, nil)
if err != nil {
return nil, fmt.Errorf("failed to convert decoded object into a v1 ConfigMap, error: %v", err)
}
return NewConfigMapCheckpoint(cm)
}
// EqualCheckpoints compares two Checkpoints for equality, if their underlying objects are equal, so are the Checkpoints
func EqualCheckpoints(a, b Checkpoint) bool {
if a != nil && b != nil {
return apiequality.Semantic.DeepEqual(a.object(), b.object())
}
if a == nil && b == nil {
return true
}
return false
}

View File

@ -1,89 +0,0 @@
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package checkpoint
import (
"testing"
"github.com/davecgh/go-spew/spew"
apiv1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
)
func TestDecodeCheckpoint(t *testing.T) {
// generate correct Checkpoint for v1/ConfigMap test case
cm, err := NewConfigMapCheckpoint(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: types.UID("uid")}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// generate unsupported object encoding for unsupported type test case
unsupported := newUnsupportedEncoded(t)
// test cases
cases := []struct {
desc string
data []byte
expect Checkpoint // expect a deeply-equal Checkpoint to be returned from Decode
err string // expect error to contain this substring
}{
// v1/ConfigMap
{"v1/ConfigMap", []byte(`{"apiVersion": "v1","kind": "ConfigMap","metadata": {"uid": "uid"}}`), cm, ""},
// malformed
{"malformed", []byte("malformed"), nil, "failed to decode"},
// no UID
{"no UID", []byte(`{"apiVersion": "v1","kind": "ConfigMap"}`), nil, "ConfigMap must have a UID"},
// well-formed, but unsupported type
{"well-formed, but unsupported encoded type", unsupported, nil, "failed to convert"},
}
for _, c := range cases {
cpt, err := DecodeCheckpoint(c.data)
if utiltest.SkipRest(t, c.desc, err, c.err) {
continue
}
// Unfortunately reflect.DeepEqual treats nil data structures as != empty data structures, so
// we have to settle for semantic equality of the underlying checkpointed API objects.
// If additional fields are added to the object that implements the Checkpoint interface,
// they should be added to a named sub-object to facilitate a DeepEquals comparison
// of the extra fields.
// decoded checkpoint should match expected checkpoint
if !apiequality.Semantic.DeepEqual(cpt.object(), c.expect.object()) {
t.Errorf("case %q, expect checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
}
}
}
// newUnsupportedEncoded returns an encoding of an object that does not have a Checkpoint implementation
func newUnsupportedEncoded(t *testing.T) []byte {
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
if err != nil {
t.Fatalf("could not create an encoder, error: %v", err)
}
unsupported := &apiv1.Node{}
data, err := runtime.Encode(encoder, unsupported)
if err != nil {
t.Fatalf("could not encode object, error: %v", err)
}
return data
}

View File

@ -20,75 +20,42 @@ import (
"fmt"
apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
)
const configMapConfigKey = "kubelet"
// configMapCheckpoint implements Checkpoint, backed by a v1/ConfigMap config source object
type configMapCheckpoint struct {
kubeletCodecs *serializer.CodecFactory // codecs for the KubeletConfiguration
configMap *apiv1.ConfigMap
// configMapPayload implements Payload, backed by a v1/ConfigMap config source object
type configMapPayload struct {
cm *apiv1.ConfigMap
}
// NewConfigMapCheckpoint returns a Checkpoint backed by `cm`. `cm` must be non-nil
// and have a non-empty ObjectMeta.UID, or an error will be returned.
func NewConfigMapCheckpoint(cm *apiv1.ConfigMap) (Checkpoint, error) {
var _ Payload = (*configMapPayload)(nil)
// NewConfigMapPayload constructs a Payload backed by a ConfigMap, which must have a non-empty UID
func NewConfigMapPayload(cm *apiv1.ConfigMap) (Payload, error) {
if cm == nil {
return nil, fmt.Errorf("ConfigMap must be non-nil to be treated as a Checkpoint")
} else if len(cm.ObjectMeta.UID) == 0 {
return nil, fmt.Errorf("ConfigMap must have a UID to be treated as a Checkpoint")
return nil, fmt.Errorf("ConfigMap must be non-nil")
} else if cm.ObjectMeta.UID == "" {
return nil, fmt.Errorf("ConfigMap must have a non-empty UID")
} else if cm.ObjectMeta.ResourceVersion == "" {
return nil, fmt.Errorf("ConfigMap must have a non-empty ResourceVersion")
}
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
return nil, err
}
return &configMapCheckpoint{kubeletCodecs, cm}, nil
return &configMapPayload{cm}, nil
}
// UID returns the UID of a configMapCheckpoint
func (c *configMapCheckpoint) UID() string {
return string(c.configMap.UID)
func (p *configMapPayload) UID() string {
return string(p.cm.UID)
}
// Parse extracts the KubeletConfiguration from v1/ConfigMap checkpoints, applies defaults, and converts to the internal type
func (c *configMapCheckpoint) Parse() (*kubeletconfig.KubeletConfiguration, error) {
const emptyCfgErr = "config was empty, but some parameters are required"
if len(c.configMap.Data) == 0 {
return nil, fmt.Errorf(emptyCfgErr)
}
config, ok := c.configMap.Data[configMapConfigKey]
if !ok {
return nil, fmt.Errorf("key %q not found in ConfigMap", configMapConfigKey)
} else if len(config) == 0 {
return nil, fmt.Errorf(emptyCfgErr)
}
return utilcodec.DecodeKubeletConfiguration(c.kubeletCodecs, []byte(config))
func (p *configMapPayload) ResourceVersion() string {
return p.cm.ResourceVersion
}
// Encode encodes a configMapCheckpoint
func (c *configMapCheckpoint) Encode() ([]byte, error) {
cm := c.configMap
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
if err != nil {
return nil, err
}
data, err := runtime.Encode(encoder, cm)
if err != nil {
return nil, err
}
return data, nil
func (p *configMapPayload) Files() map[string]string {
return p.cm.Data
}
func (c *configMapCheckpoint) object() interface{} {
return c.configMap
func (p *configMapPayload) object() interface{} {
return p.cm
}

View File

@ -17,7 +17,7 @@ limitations under the License.
package checkpoint
import (
"fmt"
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
@ -25,213 +25,124 @@ import (
apiv1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
)
func TestNewConfigMapCheckpoint(t *testing.T) {
func TestNewConfigMapPayload(t *testing.T) {
cases := []struct {
desc string
cm *apiv1.ConfigMap
err string
}{
{"nil v1/ConfigMap", nil, "must be non-nil"},
{"empty v1/ConfigMap", &apiv1.ConfigMap{}, "must have a UID"},
{"populated v1/ConfigMap",
&apiv1.ConfigMap{
{
desc: "nil",
cm: nil,
err: "ConfigMap must be non-nil",
},
{
desc: "missing uid",
cm: &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "name",
ResourceVersion: "rv",
},
},
err: "ConfigMap must have a non-empty UID",
},
{
desc: "missing resourceVersion",
cm: &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "name",
UID: types.UID("uid"),
UID: "uid",
},
},
err: "ConfigMap must have a non-empty ResourceVersion",
},
{
desc: "populated v1/ConfigMap",
cm: &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "name",
UID: "uid",
ResourceVersion: "rv",
},
Data: map[string]string{
"key1": "value1",
"key2": "value2",
},
}, ""},
}
for _, c := range cases {
cpt, err := NewConfigMapCheckpoint(c.cm)
if utiltest.SkipRest(t, c.desc, err, c.err) {
continue
}
// underlying object should match the object passed in
if !apiequality.Semantic.DeepEqual(cpt.object(), c.cm) {
t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.cm), spew.Sdump(cpt))
}
}
}
func TestConfigMapCheckpointUID(t *testing.T) {
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
for _, uidIn := range cases {
cpt := &configMapCheckpoint{
kubeletCodecs,
&apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{UID: types.UID(uidIn)},
},
}
// UID method should return the correct value of the UID
uidOut := cpt.UID()
if uidIn != uidOut {
t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
}
err: "",
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
payload, err := NewConfigMapPayload(c.cm)
utiltest.ExpectError(t, err, c.err)
if err != nil {
return
}
// underlying object should match the object passed in
if !apiequality.Semantic.DeepEqual(c.cm, payload.object()) {
t.Errorf("expect %s but got %s", spew.Sdump(c.cm), spew.Sdump(payload))
}
})
}
}
func TestConfigMapCheckpointParse(t *testing.T) {
kubeletScheme, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
func TestConfigMapPayloadUID(t *testing.T) {
const expect = "uid"
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: expect, ResourceVersion: "rv"}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
t.Fatalf("error constructing payload: %v", err)
}
uid := payload.UID()
if expect != uid {
t.Errorf("expect %q, but got %q", expect, uid)
}
}
// get the built-in default configuration
external := &kubeletconfigv1beta1.KubeletConfiguration{}
kubeletScheme.Default(external)
defaultConfig := &kubeletconfig.KubeletConfiguration{}
err = kubeletScheme.Convert(external, defaultConfig, nil)
func TestConfigMapPayloadResourceVersion(t *testing.T) {
const expect = "rv"
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: "uid", ResourceVersion: expect}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
t.Fatalf("error constructing payload: %v", err)
}
resourceVersion := payload.ResourceVersion()
if expect != resourceVersion {
t.Errorf("expect %q, but got %q", expect, resourceVersion)
}
}
func TestConfigMapPayloadFiles(t *testing.T) {
cases := []struct {
desc string
cm *apiv1.ConfigMap
expect *kubeletconfig.KubeletConfiguration
err string
data map[string]string
expect map[string]string
}{
{"empty data", &apiv1.ConfigMap{}, nil, "config was empty"},
// missing kubelet key
{"missing kubelet key", &apiv1.ConfigMap{Data: map[string]string{
"bogus": "stuff"}}, nil, fmt.Sprintf("key %q not found", configMapConfigKey)},
// invalid format
{"invalid yaml", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": "*"}}, nil, "failed to decode"},
{"invalid json", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": "{*"}}, nil, "failed to decode"},
// invalid object
{"missing kind", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `{"apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, nil, "failed to decode"},
{"missing version", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `{"kind":"KubeletConfiguration"}`}}, nil, "failed to decode"},
{"unregistered kind", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `{"kind":"BogusKind","apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, nil, "failed to decode"},
{"unregistered version", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"bogusversion"}`}}, nil, "failed to decode"},
// empty object with correct kind and version should result in the defaults for that kind and version
{"default from yaml", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1`}}, defaultConfig, ""},
{"default from json", &apiv1.ConfigMap{Data: map[string]string{
"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, defaultConfig, ""},
{"nil", nil, nil},
{"empty", map[string]string{}, map[string]string{}},
{"populated",
map[string]string{
"foo": "1",
"bar": "2",
},
map[string]string{
"foo": "1",
"bar": "2",
}},
}
for _, c := range cases {
cpt := &configMapCheckpoint{kubeletCodecs, c.cm}
kc, err := cpt.Parse()
if utiltest.SkipRest(t, c.desc, err, c.err) {
continue
}
// we expect the parsed configuration to match what we described in the ConfigMap
if !apiequality.Semantic.DeepEqual(c.expect, kc) {
t.Errorf("case %q, expect config %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(kc))
}
}
}
func TestConfigMapCheckpointEncode(t *testing.T) {
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// only one case, based on output from the existing encoder, and since
// this is hard to test (key order isn't guaranteed), we should probably
// just stick to this test case and mostly rely on the round-trip test.
cases := []struct {
desc string
cpt *configMapCheckpoint
expect string
}{
// we expect Checkpoints to be encoded as a json representation of the underlying API object
{"one-key",
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "one-key"},
Data: map[string]string{"one": ""}}},
`{"kind":"ConfigMap","apiVersion":"v1","metadata":{"name":"one-key","creationTimestamp":null},"data":{"one":""}}
`},
}
for _, c := range cases {
data, err := c.cpt.Encode()
// we don't expect any errors from encoding
if utiltest.SkipRest(t, c.desc, err, "") {
continue
}
if string(data) != c.expect {
t.Errorf("case %q, expect encoding %q but got %q", c.desc, c.expect, string(data))
}
}
}
func TestConfigMapCheckpointRoundTrip(t *testing.T) {
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
cases := []struct {
desc string
cpt *configMapCheckpoint
decodeErr string
}{
// empty data
{"empty data",
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "empty-data-sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
UID: "uid",
},
Data: map[string]string{}}},
""},
// two keys
{"two keys",
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
UID: "uid",
},
Data: map[string]string{"one": "", "two": "2"}}},
""},
// missing uid
{"missing uid",
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
UID: "",
},
Data: map[string]string{"one": "", "two": "2"}}},
"must have a UID"},
}
for _, c := range cases {
// we don't expect any errors from encoding
data, err := c.cpt.Encode()
if utiltest.SkipRest(t, c.desc, err, "") {
continue
}
after, err := DecodeCheckpoint(data)
if utiltest.SkipRest(t, c.desc, err, c.decodeErr) {
continue
}
if !apiequality.Semantic.DeepEqual(c.cpt.object(), after.object()) {
t.Errorf("case %q, expect round-trip result %s but got %s", c.desc, spew.Sdump(c.cpt), spew.Sdump(after))
}
t.Run(c.desc, func(t *testing.T) {
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: "uid", ResourceVersion: "rv"}, Data: c.data})
if err != nil {
t.Fatalf("error constructing payload: %v", err)
}
files := payload.Files()
if !reflect.DeepEqual(c.expect, files) {
t.Errorf("expected %v, but got %v", c.expect, files)
}
})
}
}

View File

@ -18,89 +18,130 @@ package checkpoint
import (
"fmt"
"math/rand"
"time"
apiv1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
kuberuntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/watch"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/client-go/tools/cache"
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
)
// RemoteConfigSource represents a remote config source object that can be downloaded as a Checkpoint
type RemoteConfigSource interface {
// UID returns the UID of the remote config source object
// Payload represents a local copy of a config source (payload) object
type Payload interface {
// UID returns a globally unique (space and time) identifier for the payload.
// The return value is guaranteed non-empty.
UID() string
// APIPath returns the API path to the remote resource, e.g. its SelfLink
APIPath() string
// Download downloads the remote config source object returns a Checkpoint backed by the object,
// or a sanitized failure reason and error if the download fails
Download(client clientset.Interface) (Checkpoint, string, error)
// Encode returns a []byte representation of the object behind the RemoteConfigSource
Encode() ([]byte, error)
// object returns the underlying source object. If you want to compare sources for equality, use EqualRemoteConfigSources,
// which compares the underlying source objects for semantic API equality.
// ResourceVersion returns a resource version for the payload.
// The return value is guaranteed non-empty.
ResourceVersion() string
// Files returns a map of filenames to file contents.
Files() map[string]string
// object returns the underlying checkpointed object.
object() interface{}
}
// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object, or returns
// a sanitized failure reason and an error if the `source` is blatantly invalid.
// RemoteConfigSource represents a remote config source object that can be downloaded as a Checkpoint
type RemoteConfigSource interface {
// KubeletFilename returns the name of the Kubelet config file as it should appear in the keys of Payload.Files()
KubeletFilename() string
// APIPath returns the API path to the remote resource, e.g. its SelfLink
APIPath() string
// UID returns the globally unique identifier for the most recently downloaded payload targeted by the source.
UID() string
// ResourceVersion returns the resource version of the most recently downloaded payload targeted by the source.
ResourceVersion() string
// Download downloads the remote config source's target object and returns a Payload backed by the object,
// or a sanitized failure reason and error if the download fails.
// Download takes an optional store as an argument. If provided, Download will check this store for the
// target object prior to contacting the API server.
// Download updates the local UID and ResourceVersion tracked by this source, based on the downloaded payload.
Download(client clientset.Interface, store cache.Store) (Payload, string, error)
// Informer returns an informer that can be used to detect changes to the remote config source
Informer(client clientset.Interface, handler cache.ResourceEventHandlerFuncs) cache.SharedInformer
// Encode returns a []byte representation of the object behind the RemoteConfigSource
Encode() ([]byte, error)
// NodeConfigSource returns a copy of the underlying apiv1.NodeConfigSource object.
// All RemoteConfigSources are expected to be backed by a NodeConfigSource,
// though the convenience methods on the interface will target the source
// type that was detected in a call to NewRemoteConfigSource.
NodeConfigSource() *apiv1.NodeConfigSource
}
// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object
// You should only call this with a non-nil config source.
// Note that the API server validates Node.Spec.ConfigSource.
func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource, string, error) {
// exactly one subfield of the config source must be non-nil, toady ConfigMapRef is the only reference
if source.ConfigMapRef == nil {
return nil, status.FailSyncReasonAllNilSubfields, fmt.Errorf("%s, NodeConfigSource was: %#v", status.FailSyncReasonAllNilSubfields, source)
// NOTE: Even though the API server validates the config, we check whether all *known* fields are
// nil here, so that if a new API server allows a new config source type, old clients can send
// an error message rather than crashing due to a nil pointer dereference.
// Exactly one reference subfield of the config source must be non-nil.
// Currently ConfigMap is the only reference subfield.
if source.ConfigMap == nil {
return nil, status.AllNilSubfieldsError, fmt.Errorf("%s, NodeConfigSource was: %#v", status.AllNilSubfieldsError, source)
}
// validate the NodeConfigSource:
// at this point we know we're using the ConfigMapRef subfield
ref := source.ConfigMapRef
// name, namespace, and UID must all be non-empty for ConfigMapRef
if ref.Name == "" || ref.Namespace == "" || string(ref.UID) == "" {
return nil, status.FailSyncReasonPartialObjectReference, fmt.Errorf("%s, ObjectReference was: %#v", status.FailSyncReasonPartialObjectReference, ref)
}
return &remoteConfigMap{source}, "", nil
}
// DecodeRemoteConfigSource is a helper for using the apimachinery to decode serialized RemoteConfigSources;
// e.g. the objects stored in the .cur and .lkg files by checkpoint/store/fsstore.go
// e.g. the metadata stored by checkpoint/store/fsstore.go
func DecodeRemoteConfigSource(data []byte) (RemoteConfigSource, error) {
// decode the remote config source
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data)
_, codecs, err := scheme.NewSchemeAndCodecs()
if err != nil {
return nil, err
}
obj, err := runtime.Decode(codecs.UniversalDecoder(), data)
if err != nil {
return nil, fmt.Errorf("failed to decode, error: %v", err)
}
// for now we assume we are trying to load an apiv1.NodeConfigSource,
// for now we assume we are trying to load an kubeletconfigv1beta1.SerializedNodeConfigSource,
// this may need to be extended if e.g. a new version of the api is born
// convert it to the external NodeConfigSource type, so we're consistently working with the external type outside of the on-disk representation
cs := &apiv1.NodeConfigSource{}
err = legacyscheme.Scheme.Convert(obj, cs, nil)
if err != nil {
return nil, fmt.Errorf("failed to convert decoded object into a v1 NodeConfigSource, error: %v", err)
cs, ok := obj.(*kubeletconfiginternal.SerializedNodeConfigSource)
if !ok {
return nil, fmt.Errorf("failed to cast decoded remote config source to *k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig.SerializedNodeConfigSource")
}
source, _, err := NewRemoteConfigSource(cs)
return source, err
// we use the v1.NodeConfigSource type on internal and external, so no need to convert to external here
source, _, err := NewRemoteConfigSource(&cs.Source)
if err != nil {
return nil, err
}
return source, nil
}
// EqualRemoteConfigSources is a helper for comparing remote config sources by
// comparing the underlying API objects for semantic equality.
func EqualRemoteConfigSources(a, b RemoteConfigSource) bool {
if a != nil && b != nil {
return apiequality.Semantic.DeepEqual(a.object(), b.object())
return apiequality.Semantic.DeepEqual(a.NodeConfigSource(), b.NodeConfigSource())
}
if a == nil && b == nil {
return true
}
return false
return a == b
}
// remoteConfigMap implements RemoteConfigSource for v1/ConfigMap config sources
@ -108,58 +149,125 @@ type remoteConfigMap struct {
source *apiv1.NodeConfigSource
}
func (r *remoteConfigMap) UID() string {
return string(r.source.ConfigMapRef.UID)
var _ RemoteConfigSource = (*remoteConfigMap)(nil)
func (r *remoteConfigMap) KubeletFilename() string {
return r.source.ConfigMap.KubeletConfigKey
}
const configMapAPIPathFmt = "/api/v1/namespaces/%s/configmaps/%s"
func (r *remoteConfigMap) APIPath() string {
ref := r.source.ConfigMapRef
ref := r.source.ConfigMap
return fmt.Sprintf(configMapAPIPathFmt, ref.Namespace, ref.Name)
}
func (r *remoteConfigMap) Download(client clientset.Interface) (Checkpoint, string, error) {
var reason string
uid := string(r.source.ConfigMapRef.UID)
func (r *remoteConfigMap) UID() string {
return string(r.source.ConfigMap.UID)
}
utillog.Infof("attempting to download ConfigMap with UID %q", uid)
func (r *remoteConfigMap) ResourceVersion() string {
return r.source.ConfigMap.ResourceVersion
}
// get the ConfigMap via namespace/name, there doesn't seem to be a way to get it by UID
cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMapRef.Namespace).Get(r.source.ConfigMapRef.Name, metav1.GetOptions{})
func (r *remoteConfigMap) Download(client clientset.Interface, store cache.Store) (Payload, string, error) {
var (
cm *apiv1.ConfigMap
err error
)
// check the in-memory store for the ConfigMap, so we can skip unnecessary downloads
if store != nil {
utillog.Infof("checking in-memory store for %s", r.APIPath())
cm, err = getConfigMapFromStore(store, r.source.ConfigMap.Namespace, r.source.ConfigMap.Name)
if err != nil {
// just log the error, we'll attempt a direct download instead
utillog.Errorf("failed to check in-memory store for %s, error: %v", r.APIPath(), err)
} else if cm != nil {
utillog.Infof("found %s in in-memory store, UID: %s, ResourceVersion: %s", r.APIPath(), cm.UID, cm.ResourceVersion)
} else {
utillog.Infof("did not find %s in in-memory store", r.APIPath())
}
}
// if we didn't find the ConfigMap in the in-memory store, download it from the API server
if cm == nil {
utillog.Infof("attempting to download %s", r.APIPath())
cm, err = client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Get(r.source.ConfigMap.Name, metav1.GetOptions{})
if err != nil {
return nil, status.DownloadError, fmt.Errorf("%s, error: %v", status.DownloadError, err)
}
utillog.Infof("successfully downloaded %s, UID: %s, ResourceVersion: %s", r.APIPath(), cm.UID, cm.ResourceVersion)
} // Assert: Now we have a non-nil ConfigMap
// construct Payload from the ConfigMap
payload, err := NewConfigMapPayload(cm)
if err != nil {
reason = fmt.Sprintf(status.FailSyncReasonDownloadFmt, r.APIPath())
return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
// We only expect an error here if ObjectMeta is lacking UID or ResourceVersion. This should
// never happen on objects in the informer's store, or objects downloaded from the API server
// directly, so we report InternalError.
return nil, status.InternalError, fmt.Errorf("%s, error: %v", status.InternalError, err)
}
// update internal UID and ResourceVersion based on latest ConfigMap
r.source.ConfigMap.UID = cm.UID
r.source.ConfigMap.ResourceVersion = cm.ResourceVersion
return payload, "", nil
}
func (r *remoteConfigMap) Informer(client clientset.Interface, handler cache.ResourceEventHandlerFuncs) cache.SharedInformer {
// select ConfigMap by name
fieldselector := fields.OneTermEqualSelector("metadata.name", r.source.ConfigMap.Name)
// add some randomness to resync period, which can help avoid controllers falling into lock-step
minResyncPeriod := 15 * time.Minute
factor := rand.Float64() + 1
resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor)
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (kuberuntime.Object, error) {
return client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).List(metav1.ListOptions{
FieldSelector: fieldselector.String(),
})
},
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
return client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Watch(metav1.ListOptions{
FieldSelector: fieldselector.String(),
ResourceVersion: options.ResourceVersion,
})
},
}
// ensure that UID matches the UID on the reference, the ObjectReference must be unambiguous
if r.source.ConfigMapRef.UID != cm.UID {
reason = fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, r.source.ConfigMapRef.UID, r.APIPath(), cm.UID)
return nil, reason, fmt.Errorf(reason)
}
informer := cache.NewSharedInformer(lw, &apiv1.ConfigMap{}, resyncPeriod)
informer.AddEventHandler(handler)
checkpoint, err := NewConfigMapCheckpoint(cm)
if err != nil {
reason = fmt.Sprintf("invalid downloaded object")
return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
}
utillog.Infof("successfully downloaded ConfigMap with UID %q", uid)
return checkpoint, "", nil
return informer
}
func (r *remoteConfigMap) Encode() ([]byte, error) {
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
encoder, err := utilcodec.NewKubeletconfigYAMLEncoder(kubeletconfigv1beta1.SchemeGroupVersion)
if err != nil {
return nil, err
}
data, err := runtime.Encode(encoder, r.source)
data, err := runtime.Encode(encoder, &kubeletconfigv1beta1.SerializedNodeConfigSource{Source: *r.source})
if err != nil {
return nil, err
}
return data, nil
}
func (r *remoteConfigMap) object() interface{} {
return r.source
func (r *remoteConfigMap) NodeConfigSource() *apiv1.NodeConfigSource {
return r.source.DeepCopy()
}
func getConfigMapFromStore(store cache.Store, namespace, name string) (*apiv1.ConfigMap, error) {
key := fmt.Sprintf("%s/%s", namespace, name)
obj, ok, err := store.GetByKey(key)
if err != nil || !ok {
return nil, err
}
cm, ok := obj.(*apiv1.ConfigMap)
if !ok {
err := fmt.Errorf("failed to cast object %s from informer's store to ConfigMap", key)
utillog.Errorf(err.Error())
return nil, err
}
return cm, nil
}

View File

@ -25,9 +25,9 @@ import (
apiv1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
clientset "k8s.io/client-go/kubernetes"
fakeclient "k8s.io/client-go/kubernetes/fake"
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
"k8s.io/client-go/tools/cache"
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
)
@ -38,118 +38,205 @@ func TestNewRemoteConfigSource(t *testing.T) {
expect RemoteConfigSource
err string
}{
// all NodeConfigSource subfields nil
{"all NodeConfigSource subfields nil",
&apiv1.NodeConfigSource{}, nil, "exactly one subfield must be non-nil"},
{"ConfigMapRef: empty name, namespace, and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty name and namespace
{"ConfigMapRef: empty name and namespace",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "uid"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty name and UID
{"ConfigMapRef: empty name and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty namespace and UID
{"ConfigMapRef: empty namespace and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty UID
{"ConfigMapRef: empty namespace and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty namespace
{"ConfigMapRef: empty namespace and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", UID: "uid"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: empty name
{"ConfigMapRef: empty namespace and UID",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace", UID: "uid"}}, nil, "invalid ObjectReference"},
// ConfigMapRef: valid reference
{"ConfigMapRef: valid reference",
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}},
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}}, ""},
{
desc: "all NodeConfigSource subfields nil",
source: &apiv1.NodeConfigSource{},
expect: nil,
err: "exactly one subfield must be non-nil",
},
{
desc: "ConfigMap: valid reference",
source: &apiv1.NodeConfigSource{
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "name",
Namespace: "namespace",
UID: "uid",
KubeletConfigKey: "kubelet",
}},
expect: &remoteConfigMap{&apiv1.NodeConfigSource{
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "name",
Namespace: "namespace",
UID: "uid",
KubeletConfigKey: "kubelet",
}}},
err: "",
},
}
for _, c := range cases {
src, _, err := NewRemoteConfigSource(c.source)
if utiltest.SkipRest(t, c.desc, err, c.err) {
continue
}
// underlying object should match the object passed in
if !apiequality.Semantic.DeepEqual(c.expect.object(), src.object()) {
t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(src))
}
t.Run(c.desc, func(t *testing.T) {
source, _, err := NewRemoteConfigSource(c.source)
utiltest.ExpectError(t, err, c.err)
if err != nil {
return
}
// underlying object should match the object passed in
if !apiequality.Semantic.DeepEqual(c.expect.NodeConfigSource(), source.NodeConfigSource()) {
t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(source))
}
})
}
}
func TestRemoteConfigMapUID(t *testing.T) {
cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
for _, uidIn := range cases {
cpt := &remoteConfigMap{
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: types.UID(uidIn)}},
}
// UID method should return the correct value of the UID
uidOut := cpt.UID()
if uidIn != uidOut {
t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
}
const expect = "uid"
source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "name",
Namespace: "namespace",
UID: expect,
KubeletConfigKey: "kubelet",
}})
if err != nil {
t.Fatalf("error constructing remote config source: %v", err)
}
uid := source.UID()
if expect != uid {
t.Errorf("expect %q, but got %q", expect, uid)
}
}
func TestRemoteConfigMapAPIPath(t *testing.T) {
name := "name"
namespace := "namespace"
cpt := &remoteConfigMap{
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: name, Namespace: namespace, UID: ""}},
const (
name = "name"
namespace = "namespace"
)
source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: name,
Namespace: namespace,
UID: "uid",
KubeletConfigKey: "kubelet",
}})
if err != nil {
t.Fatalf("error constructing remote config source: %v", err)
}
expect := fmt.Sprintf(configMapAPIPathFmt, cpt.source.ConfigMapRef.Namespace, cpt.source.ConfigMapRef.Name)
// APIPath() method should return the correct path to the referenced resource
path := cpt.APIPath()
expect := fmt.Sprintf(configMapAPIPathFmt, namespace, name)
path := source.APIPath()
if expect != path {
t.Errorf("expect APIPath() to return %q, but got %q", expect, namespace)
t.Errorf("expect %q, but got %q", expect, path)
}
}
func TestRemoteConfigMapDownload(t *testing.T) {
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
cm := &apiv1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "name",
Namespace: "namespace",
UID: "uid",
Name: "name",
Namespace: "namespace",
UID: "uid",
ResourceVersion: "1",
}}
client := fakeclient.NewSimpleClientset(cm)
source := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "name",
Namespace: "namespace",
KubeletConfigKey: "kubelet",
}}
expectPayload, err := NewConfigMapPayload(cm)
if err != nil {
t.Fatalf("error constructing payload: %v", err)
}
missingStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
hasStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
if err := hasStore.Add(cm); err != nil {
t.Fatalf("unexpected error constructing hasStore")
}
missingClient := fakeclient.NewSimpleClientset()
hasClient := fakeclient.NewSimpleClientset(cm)
cases := []struct {
desc string
source RemoteConfigSource
expect Checkpoint
client clientset.Interface
store cache.Store
err string
}{
// object doesn't exist
{"object doesn't exist",
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "bogus", Namespace: "namespace", UID: "bogus"}}},
nil, "not found"},
// UID of downloaded object doesn't match UID of referent found via namespace/name
{"UID is incorrect for namespace/name",
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "bogus"}}},
nil, "does not match"},
// successful download
{"object exists and reference is correct",
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}},
&configMapCheckpoint{kubeletCodecs, cm}, ""},
{
desc: "nil store, object does not exist in API server",
client: missingClient,
err: "not found",
},
{
desc: "nil store, object exists in API server",
client: hasClient,
},
{
desc: "object exists in store and API server",
store: hasStore,
client: hasClient,
},
{
desc: "object exists in store, but does not exist in API server",
store: hasStore,
client: missingClient,
},
{
desc: "object does not exist in store, but exists in API server",
store: missingStore,
client: hasClient,
},
{
desc: "object does not exist in store or API server",
client: missingClient,
store: missingStore,
err: "not found",
},
}
for _, c := range cases {
cpt, _, err := c.source.Download(client)
if utiltest.SkipRest(t, c.desc, err, c.err) {
continue
}
// "downloaded" object should match the expected
if !apiequality.Semantic.DeepEqual(c.expect.object(), cpt.object()) {
t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
}
t.Run(c.desc, func(t *testing.T) {
// deep copy so we can always check the UID/ResourceVersion are set after Download
s, _, err := NewRemoteConfigSource(source.DeepCopy())
if err != nil {
t.Fatalf("error constructing remote config source %v", err)
}
// attempt download
p, _, err := s.Download(c.client, c.store)
utiltest.ExpectError(t, err, c.err)
if err != nil {
return
}
// downloaded object should match the expected
if !apiequality.Semantic.DeepEqual(expectPayload.object(), p.object()) {
t.Errorf("expect Checkpoint %s but got %s", spew.Sdump(expectPayload), spew.Sdump(p))
}
// source UID and ResourceVersion should be updated by Download
if p.UID() != s.UID() {
t.Errorf("expect UID to be updated by Download to match payload: %s, but got source UID: %s", p.UID(), s.UID())
}
if p.ResourceVersion() != s.ResourceVersion() {
t.Errorf("expect ResourceVersion to be updated by Download to match payload: %s, but got source ResourceVersion: %s", p.ResourceVersion(), s.ResourceVersion())
}
})
}
}
func TestEqualRemoteConfigSources(t *testing.T) {
cases := []struct {
desc string
a RemoteConfigSource
b RemoteConfigSource
expect bool
}{
{"both nil", nil, nil, true},
{"a nil", nil, &remoteConfigMap{}, false},
{"b nil", &remoteConfigMap{}, nil, false},
{"neither nil, equal", &remoteConfigMap{}, &remoteConfigMap{}, true},
{
desc: "neither nil, not equal",
a: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{Name: "a"}}},
b: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{KubeletConfigKey: "kubelet"}}},
expect: false,
},
}
for _, c := range cases {
t.Run(c.desc, func(t *testing.T) {
if EqualRemoteConfigSources(c.a, c.b) != c.expect {
t.Errorf("expected EqualRemoteConfigSources to return %t, but got %t", c.expect, !c.expect)
}
})
}
}

View File

@ -14,7 +14,11 @@ go_test(
],
embed = [":go_default_library"],
deps = [
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
"//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
"//pkg/kubelet/kubeletconfig/util/files:go_default_library",
"//pkg/kubelet/kubeletconfig/util/test:go_default_library",
"//pkg/util/filesystem:go_default_library",
@ -34,7 +38,9 @@ go_library(
],
importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store",
deps = [
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
"//pkg/kubelet/kubeletconfig/configfiles:go_default_library",
"//pkg/kubelet/kubeletconfig/util/files:go_default_library",
"//pkg/kubelet/kubeletconfig/util/log:go_default_library",
"//pkg/util/filesystem:go_default_library",

View File

@ -20,52 +20,51 @@ import (
"fmt"
"time"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
)
// so far only implements Current(), LastKnownGood(), SetCurrent(), and SetLastKnownGood()
// so far only implements Assigned(), LastKnownGood(), SetAssigned(), and SetLastKnownGood()
type fakeStore struct {
current checkpoint.RemoteConfigSource
assigned checkpoint.RemoteConfigSource
lastKnownGood checkpoint.RemoteConfigSource
}
var _ Store = (*fakeStore)(nil)
func (s *fakeStore) Initialize() error {
return fmt.Errorf("Initialize method not supported")
}
func (s *fakeStore) Exists(uid string) (bool, error) {
func (s *fakeStore) Exists(source checkpoint.RemoteConfigSource) (bool, error) {
return false, fmt.Errorf("Exists method not supported")
}
func (s *fakeStore) Save(c checkpoint.Checkpoint) error {
func (s *fakeStore) Save(c checkpoint.Payload) error {
return fmt.Errorf("Save method not supported")
}
func (s *fakeStore) Load(uid string) (checkpoint.Checkpoint, error) {
func (s *fakeStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error) {
return nil, fmt.Errorf("Load method not supported")
}
func (s *fakeStore) CurrentModified() (time.Time, error) {
return time.Time{}, fmt.Errorf("CurrentModified method not supported")
func (s *fakeStore) AssignedModified() (time.Time, error) {
return time.Time{}, fmt.Errorf("AssignedModified method not supported")
}
func (s *fakeStore) Current() (checkpoint.RemoteConfigSource, error) {
return s.current, nil
func (s *fakeStore) Assigned() (checkpoint.RemoteConfigSource, error) {
return s.assigned, nil
}
func (s *fakeStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
return s.lastKnownGood, nil
}
func (s *fakeStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
s.current = source
func (s *fakeStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
s.assigned = source
return nil
}
func (s *fakeStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
return setCurrentUpdated(s, source)
}
func (s *fakeStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
s.lastKnownGood = source
return nil

View File

@ -21,82 +21,112 @@ import (
"path/filepath"
"time"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
)
const (
curFile = ".cur"
lkgFile = ".lkg"
metaDir = "meta"
assignedFile = "assigned"
lastKnownGoodFile = "last-known-good"
checkpointsDir = "checkpoints"
)
// fsStore is for tracking checkpoints in the local filesystem, implements Store
type fsStore struct {
// fs is the filesystem to use for storage operations; can be mocked for testing
fs utilfs.Filesystem
// checkpointsDir is the absolute path to the storage directory for fsStore
checkpointsDir string
// dir is the absolute path to the storage directory for fsStore
dir string
}
// NewFsStore returns a Store that saves its data in `checkpointsDir`
func NewFsStore(fs utilfs.Filesystem, checkpointsDir string) Store {
var _ Store = (*fsStore)(nil)
// NewFsStore returns a Store that saves its data in dir
func NewFsStore(fs utilfs.Filesystem, dir string) Store {
return &fsStore{
fs: fs,
checkpointsDir: checkpointsDir,
fs: fs,
dir: dir,
}
}
func (s *fsStore) Initialize() error {
utillog.Infof("initializing config checkpoints directory %q", s.checkpointsDir)
if err := utilfiles.EnsureDir(s.fs, s.checkpointsDir); err != nil {
utillog.Infof("initializing config checkpoints directory %q", s.dir)
// ensure top-level dir for store
if err := utilfiles.EnsureDir(s.fs, s.dir); err != nil {
return err
}
if err := utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, curFile)); err != nil {
// ensure metadata directory and reference files (tracks assigned and lkg configs)
if err := utilfiles.EnsureDir(s.fs, filepath.Join(s.dir, metaDir)); err != nil {
return err
}
return utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, lkgFile))
if err := utilfiles.EnsureFile(s.fs, s.metaPath(assignedFile)); err != nil {
return err
}
if err := utilfiles.EnsureFile(s.fs, s.metaPath(lastKnownGoodFile)); err != nil {
return err
}
// ensure checkpoints directory (saves unpacked payloads in subdirectories named after payload UID)
return utilfiles.EnsureDir(s.fs, filepath.Join(s.dir, checkpointsDir))
}
func (s *fsStore) Exists(uid string) (bool, error) {
ok, err := utilfiles.FileExists(s.fs, filepath.Join(s.checkpointsDir, uid))
func (s *fsStore) Exists(source checkpoint.RemoteConfigSource) (bool, error) {
const errfmt = "failed to determine whether checkpoint exists for source %s, UID: %s, ResourceVersion: %s exists, error: %v"
if len(source.UID()) == 0 {
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), "empty UID is ambiguous")
}
if len(source.ResourceVersion()) == 0 {
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), "empty ResourceVersion is ambiguous")
}
// we check whether the directory was created for the resource
ok, err := utilfiles.DirExists(s.fs, s.checkpointPath(source.UID(), source.ResourceVersion()))
if err != nil {
return false, fmt.Errorf("failed to determine whether checkpoint %q exists, error: %v", uid, err)
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), err)
}
return ok, nil
}
func (s *fsStore) Save(c checkpoint.Checkpoint) error {
// encode the checkpoint
data, err := c.Encode()
if err != nil {
func (s *fsStore) Save(payload checkpoint.Payload) error {
// Note: Payload interface guarantees UID() and ResourceVersion() to be non-empty
path := s.checkpointPath(payload.UID(), payload.ResourceVersion())
// ensure the parent dir (checkpoints/uid) exists, since ReplaceDir requires the parent of the replacee
// to exist, and we checkpoint as checkpoints/uid/resourceVersion/files-from-configmap
if err := utilfiles.EnsureDir(s.fs, filepath.Dir(path)); err != nil {
return err
}
// save the file
return utilfiles.ReplaceFile(s.fs, filepath.Join(s.checkpointsDir, c.UID()), data)
// save the checkpoint's files in the appropriate checkpoint dir
return utilfiles.ReplaceDir(s.fs, path, payload.Files())
}
func (s *fsStore) Load(uid string) (checkpoint.Checkpoint, error) {
filePath := filepath.Join(s.checkpointsDir, uid)
utillog.Infof("loading configuration from %q", filePath)
// load the file
data, err := s.fs.ReadFile(filePath)
if err != nil {
return nil, fmt.Errorf("failed to read checkpoint file %q, error: %v", filePath, err)
func (s *fsStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error) {
sourceFmt := fmt.Sprintf("%s, UID: %s, ResourceVersion: %s", source.APIPath(), source.UID(), source.ResourceVersion())
// check if a checkpoint exists for the source
if ok, err := s.Exists(source); err != nil {
return nil, err
} else if !ok {
return nil, fmt.Errorf("no checkpoint for source %s", sourceFmt)
}
// decode it
c, err := checkpoint.DecodeCheckpoint(data)
// load the kubelet config file
utillog.Infof("loading Kubelet configuration checkpoint for source %s", sourceFmt)
loader, err := configfiles.NewFsLoader(s.fs, filepath.Join(s.checkpointPath(source.UID(), source.ResourceVersion()), source.KubeletFilename()))
if err != nil {
return nil, fmt.Errorf("failed to decode checkpoint file %q, error: %v", filePath, err)
return nil, err
}
return c, nil
kc, err := loader.Load()
if err != nil {
return nil, err
}
return kc, nil
}
func (s *fsStore) CurrentModified() (time.Time, error) {
path := filepath.Join(s.checkpointsDir, curFile)
func (s *fsStore) AssignedModified() (time.Time, error) {
path := s.metaPath(assignedFile)
info, err := s.fs.Stat(path)
if err != nil {
return time.Time{}, fmt.Errorf("failed to stat %q while checking modification time, error: %v", path, err)
@ -104,35 +134,36 @@ func (s *fsStore) CurrentModified() (time.Time, error) {
return info.ModTime(), nil
}
func (s *fsStore) Current() (checkpoint.RemoteConfigSource, error) {
return s.sourceFromFile(curFile)
func (s *fsStore) Assigned() (checkpoint.RemoteConfigSource, error) {
return readRemoteConfigSource(s.fs, s.metaPath(assignedFile))
}
func (s *fsStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
return s.sourceFromFile(lkgFile)
return readRemoteConfigSource(s.fs, s.metaPath(lastKnownGoodFile))
}
func (s *fsStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
return s.setSourceFile(curFile, source)
}
func (s *fsStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
return setCurrentUpdated(s, source)
func (s *fsStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
return writeRemoteConfigSource(s.fs, s.metaPath(assignedFile), source)
}
func (s *fsStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
return s.setSourceFile(lkgFile, source)
return writeRemoteConfigSource(s.fs, s.metaPath(lastKnownGoodFile), source)
}
func (s *fsStore) Reset() (bool, error) {
return reset(s)
}
// sourceFromFile returns the RemoteConfigSource stored in the file at `s.checkpointsDir/relPath`,
// or nil if the file is empty
func (s *fsStore) sourceFromFile(relPath string) (checkpoint.RemoteConfigSource, error) {
path := filepath.Join(s.checkpointsDir, relPath)
data, err := s.fs.ReadFile(path)
func (s *fsStore) checkpointPath(uid, resourceVersion string) string {
return filepath.Join(s.dir, checkpointsDir, uid, resourceVersion)
}
func (s *fsStore) metaPath(name string) string {
return filepath.Join(s.dir, metaDir, name)
}
func readRemoteConfigSource(fs utilfs.Filesystem, path string) (checkpoint.RemoteConfigSource, error) {
data, err := fs.ReadFile(path)
if err != nil {
return nil, err
} else if len(data) == 0 {
@ -141,17 +172,23 @@ func (s *fsStore) sourceFromFile(relPath string) (checkpoint.RemoteConfigSource,
return checkpoint.DecodeRemoteConfigSource(data)
}
// set source file replaces the file at `s.checkpointsDir/relPath` with a file containing `source`
func (s *fsStore) setSourceFile(relPath string, source checkpoint.RemoteConfigSource) error {
path := filepath.Join(s.checkpointsDir, relPath)
func writeRemoteConfigSource(fs utilfs.Filesystem, path string, source checkpoint.RemoteConfigSource) error {
// if nil, reset the file
if source == nil {
return utilfiles.ReplaceFile(s.fs, path, []byte{})
return utilfiles.ReplaceFile(fs, path, []byte{})
}
// check that UID and ResourceVersion are non-empty,
// error to save reference if the checkpoint can't be fully resolved
if source.UID() == "" {
return fmt.Errorf("failed to write RemoteConfigSource, empty UID is ambiguous")
}
if source.ResourceVersion() == "" {
return fmt.Errorf("failed to write RemoteConfigSource, empty ResourceVersion is ambiguous")
}
// encode the source and save it to the file
data, err := source.Encode()
if err != nil {
return err
}
return utilfiles.ReplaceFile(s.fs, path, data)
return utilfiles.ReplaceFile(fs, path, data)
}

File diff suppressed because it is too large Load Diff

View File

@ -20,65 +20,51 @@ import (
"fmt"
"time"
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
)
// Store saves checkpoints and information about which is the current and last-known-good checkpoint to a storage layer
// Store saves checkpoints and information about which is the assigned and last-known-good checkpoint to a storage layer
type Store interface {
// Initialize sets up the storage layer
Initialize() error
// Exists returns true if a checkpoint with `uid` exists in the store, false otherwise
Exists(uid string) (bool, error)
// Save saves the checkpoint to the storage layer
Save(c checkpoint.Checkpoint) error
// Load loads the checkpoint with UID `uid` from the storage layer, or returns an error if the checkpoint does not exist
Load(uid string) (checkpoint.Checkpoint, error)
// CurrentModified returns the last time that the current UID was set
CurrentModified() (time.Time, error)
// Current returns the source that points to the current checkpoint, or nil if no current checkpoint is set
Current() (checkpoint.RemoteConfigSource, error)
// Exists returns true if the object referenced by `source` has been checkpointed.
// The source must be unambiguous - e.g. if referencing an API object it must specify both uid and resourceVersion.
Exists(source checkpoint.RemoteConfigSource) (bool, error)
// Save Kubelet config payloads to the storage layer. It must be possible to unmarshal the payload to a KubeletConfiguration.
// The following payload types are supported:
// - k8s.io/api/core/v1.ConfigMap
Save(c checkpoint.Payload) error
// Load loads the KubeletConfiguration from the checkpoint referenced by `source`.
Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error)
// AssignedModified returns the last time that the assigned checkpoint was set
AssignedModified() (time.Time, error)
// Assigned returns the source that points to the checkpoint currently assigned to the Kubelet, or nil if no assigned checkpoint is set
Assigned() (checkpoint.RemoteConfigSource, error)
// LastKnownGood returns the source that points to the last-known-good checkpoint, or nil if no last-known-good checkpoint is set
LastKnownGood() (checkpoint.RemoteConfigSource, error)
// SetCurrent saves the source that points to the current checkpoint, set to nil to unset
SetCurrent(source checkpoint.RemoteConfigSource) error
// SetCurrentUpdated is similar to SetCurrent, but also returns whether the current checkpoint changed as a result
SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error)
// SetAssigned saves the source that points to the assigned checkpoint, set to nil to unset
SetAssigned(source checkpoint.RemoteConfigSource) error
// SetLastKnownGood saves the source that points to the last-known-good checkpoint, set to nil to unset
SetLastKnownGood(source checkpoint.RemoteConfigSource) error
// Reset unsets the current and last-known-good UIDs and returns whether the current UID was unset as a result of the reset
// Reset unsets the assigned and last-known-good checkpoints and returns whether the assigned checkpoint was unset as a result of the reset
Reset() (bool, error)
}
// reset is a helper for implementing Reset, which can be implemented in terms of Store methods
func reset(s Store) (bool, error) {
assigned, err := s.Assigned()
if err != nil {
return false, err
}
if err := s.SetLastKnownGood(nil); err != nil {
return false, fmt.Errorf("failed to reset last-known-good UID in checkpoint store, error: %v", err)
}
updated, err := s.SetCurrentUpdated(nil)
if err != nil {
return false, fmt.Errorf("failed to reset current UID in checkpoint store, error: %v", err)
if err := s.SetAssigned(nil); err != nil {
return false, fmt.Errorf("failed to reset assigned UID in checkpoint store, error: %v", err)
}
return updated, nil
}
// setCurrentUpdated is a helper for implementing SetCurrentUpdated, which can be implemented in terms of Store methods
func setCurrentUpdated(s Store, source checkpoint.RemoteConfigSource) (bool, error) {
cur, err := s.Current()
if err != nil {
return false, err
}
// if both are nil, no need to update
if cur == nil && source == nil {
return false, nil
}
// if UIDs match, no need to update
if (source != nil && cur != nil) && cur.UID() == source.UID() {
return false, nil
}
// update the source
if err := s.SetCurrent(source); err != nil {
return false, err
}
return true, nil
return assigned != nil, nil
}

View File

@ -26,11 +26,21 @@ import (
)
func TestReset(t *testing.T) {
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "name",
Namespace: "namespace",
UID: "uid",
KubeletConfigKey: "kubelet",
}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
Name: "other-name",
Namespace: "namespace",
UID: "other-uid",
KubeletConfigKey: "kubelet",
}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -38,59 +48,24 @@ func TestReset(t *testing.T) {
s *fakeStore
updated bool
}{
{&fakeStore{current: nil, lastKnownGood: nil}, false},
{&fakeStore{current: source, lastKnownGood: nil}, true},
{&fakeStore{current: nil, lastKnownGood: source}, false},
{&fakeStore{current: source, lastKnownGood: source}, true},
{&fakeStore{current: source, lastKnownGood: otherSource}, true},
{&fakeStore{current: otherSource, lastKnownGood: source}, true},
{&fakeStore{assigned: nil, lastKnownGood: nil}, false},
{&fakeStore{assigned: source, lastKnownGood: nil}, true},
{&fakeStore{assigned: nil, lastKnownGood: source}, false},
{&fakeStore{assigned: source, lastKnownGood: source}, true},
{&fakeStore{assigned: source, lastKnownGood: otherSource}, true},
{&fakeStore{assigned: otherSource, lastKnownGood: source}, true},
}
for _, c := range cases {
updated, err := reset(c.s)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if c.s.current != nil || c.s.lastKnownGood != nil {
t.Errorf("case %q, expect nil for current and last-known-good checkpoints, but still have %q and %q, respectively",
spew.Sdump(c.s), c.s.current, c.s.lastKnownGood)
if c.s.assigned != nil || c.s.lastKnownGood != nil {
t.Errorf("case %q, expect nil for assigned and last-known-good checkpoints, but still have %q and %q, respectively",
spew.Sdump(c.s), c.s.assigned, c.s.lastKnownGood)
}
if c.updated != updated {
t.Errorf("case %q, expect reset to return %t, but got %t", spew.Sdump(c.s), c.updated, updated)
}
}
}
func TestSetCurrentUpdated(t *testing.T) {
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
cases := []struct {
s *fakeStore
newCurrent checkpoint.RemoteConfigSource
updated bool
}{
{&fakeStore{current: nil}, nil, false},
{&fakeStore{current: nil}, source, true},
{&fakeStore{current: source}, source, false},
{&fakeStore{current: source}, nil, true},
{&fakeStore{current: source}, otherSource, true},
}
for _, c := range cases {
current := c.s.current
updated, err := setCurrentUpdated(c.s, c.newCurrent)
if err != nil {
t.Fatalf("case %q -> %q, unexpected error: %v", current, c.newCurrent, err)
}
if c.newCurrent != c.s.current {
t.Errorf("case %q -> %q, expect current UID to be %q, but got %q", current, c.newCurrent, c.newCurrent, c.s.current)
}
if c.updated != updated {
t.Errorf("case %q -> %q, expect setCurrentUpdated to return %t, but got %t", current, c.newCurrent, c.updated, updated)
}
}
}