mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-14 10:53:34 +00:00
vendor update for CSI 0.3.0
This commit is contained in:
16
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/BUILD
generated
vendored
16
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/BUILD
generated
vendored
@ -9,48 +9,44 @@ load(
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"checkpoint_test.go",
|
||||
"configmap_test.go",
|
||||
"download_test.go",
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/test:go_default_library",
|
||||
"//vendor/github.com/davecgh/go-spew/spew:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"checkpoint.go",
|
||||
"configmap.go",
|
||||
"download.go",
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint",
|
||||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/status:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/log:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/api/equality:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/fields:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/watch:go_default_library",
|
||||
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
|
||||
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
|
72
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go
generated
vendored
72
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/checkpoint.go
generated
vendored
@ -1,72 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
)
|
||||
|
||||
// Checkpoint represents a local copy of a config source (payload) object
|
||||
type Checkpoint interface {
|
||||
// UID returns the UID of the config source object behind the Checkpoint
|
||||
UID() string
|
||||
// Parse extracts the KubeletConfiguration from the checkpoint, applies defaults, and converts to the internal type
|
||||
Parse() (*kubeletconfig.KubeletConfiguration, error)
|
||||
// Encode returns a []byte representation of the config source object behind the Checkpoint
|
||||
Encode() ([]byte, error)
|
||||
|
||||
// object returns the underlying checkpointed object. If you want to compare sources for equality, use EqualCheckpoints,
|
||||
// which compares the underlying checkpointed objects for semantic API equality.
|
||||
object() interface{}
|
||||
}
|
||||
|
||||
// DecodeCheckpoint is a helper for using the apimachinery to decode serialized checkpoints
|
||||
func DecodeCheckpoint(data []byte) (Checkpoint, error) {
|
||||
// decode the checkpoint
|
||||
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode, error: %v", err)
|
||||
}
|
||||
|
||||
// TODO(mtaufen): for now we assume we are trying to load a ConfigMap checkpoint, may need to extend this if we allow other checkpoint types
|
||||
|
||||
// convert it to the external ConfigMap type, so we're consistently working with the external type outside of the on-disk representation
|
||||
cm := &apiv1.ConfigMap{}
|
||||
err = legacyscheme.Scheme.Convert(obj, cm, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert decoded object into a v1 ConfigMap, error: %v", err)
|
||||
}
|
||||
|
||||
return NewConfigMapCheckpoint(cm)
|
||||
}
|
||||
|
||||
// EqualCheckpoints compares two Checkpoints for equality, if their underlying objects are equal, so are the Checkpoints
|
||||
func EqualCheckpoints(a, b Checkpoint) bool {
|
||||
if a != nil && b != nil {
|
||||
return apiequality.Semantic.DeepEqual(a.object(), b.object())
|
||||
}
|
||||
if a == nil && b == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
89
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go
generated
vendored
89
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/checkpoint_test.go
generated
vendored
@ -1,89 +0,0 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
|
||||
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
|
||||
)
|
||||
|
||||
func TestDecodeCheckpoint(t *testing.T) {
|
||||
// generate correct Checkpoint for v1/ConfigMap test case
|
||||
cm, err := NewConfigMapCheckpoint(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: types.UID("uid")}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// generate unsupported object encoding for unsupported type test case
|
||||
unsupported := newUnsupportedEncoded(t)
|
||||
|
||||
// test cases
|
||||
cases := []struct {
|
||||
desc string
|
||||
data []byte
|
||||
expect Checkpoint // expect a deeply-equal Checkpoint to be returned from Decode
|
||||
err string // expect error to contain this substring
|
||||
}{
|
||||
// v1/ConfigMap
|
||||
{"v1/ConfigMap", []byte(`{"apiVersion": "v1","kind": "ConfigMap","metadata": {"uid": "uid"}}`), cm, ""},
|
||||
// malformed
|
||||
{"malformed", []byte("malformed"), nil, "failed to decode"},
|
||||
// no UID
|
||||
{"no UID", []byte(`{"apiVersion": "v1","kind": "ConfigMap"}`), nil, "ConfigMap must have a UID"},
|
||||
// well-formed, but unsupported type
|
||||
{"well-formed, but unsupported encoded type", unsupported, nil, "failed to convert"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
cpt, err := DecodeCheckpoint(c.data)
|
||||
if utiltest.SkipRest(t, c.desc, err, c.err) {
|
||||
continue
|
||||
}
|
||||
// Unfortunately reflect.DeepEqual treats nil data structures as != empty data structures, so
|
||||
// we have to settle for semantic equality of the underlying checkpointed API objects.
|
||||
// If additional fields are added to the object that implements the Checkpoint interface,
|
||||
// they should be added to a named sub-object to facilitate a DeepEquals comparison
|
||||
// of the extra fields.
|
||||
// decoded checkpoint should match expected checkpoint
|
||||
if !apiequality.Semantic.DeepEqual(cpt.object(), c.expect.object()) {
|
||||
t.Errorf("case %q, expect checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newUnsupportedEncoded returns an encoding of an object that does not have a Checkpoint implementation
|
||||
func newUnsupportedEncoded(t *testing.T) []byte {
|
||||
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
|
||||
if err != nil {
|
||||
t.Fatalf("could not create an encoder, error: %v", err)
|
||||
}
|
||||
unsupported := &apiv1.Node{}
|
||||
data, err := runtime.Encode(encoder, unsupported)
|
||||
if err != nil {
|
||||
t.Fatalf("could not encode object, error: %v", err)
|
||||
}
|
||||
return data
|
||||
}
|
75
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/configmap.go
generated
vendored
75
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/configmap.go
generated
vendored
@ -20,75 +20,42 @@ import (
|
||||
"fmt"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
|
||||
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
|
||||
)
|
||||
|
||||
const configMapConfigKey = "kubelet"
|
||||
|
||||
// configMapCheckpoint implements Checkpoint, backed by a v1/ConfigMap config source object
|
||||
type configMapCheckpoint struct {
|
||||
kubeletCodecs *serializer.CodecFactory // codecs for the KubeletConfiguration
|
||||
configMap *apiv1.ConfigMap
|
||||
// configMapPayload implements Payload, backed by a v1/ConfigMap config source object
|
||||
type configMapPayload struct {
|
||||
cm *apiv1.ConfigMap
|
||||
}
|
||||
|
||||
// NewConfigMapCheckpoint returns a Checkpoint backed by `cm`. `cm` must be non-nil
|
||||
// and have a non-empty ObjectMeta.UID, or an error will be returned.
|
||||
func NewConfigMapCheckpoint(cm *apiv1.ConfigMap) (Checkpoint, error) {
|
||||
var _ Payload = (*configMapPayload)(nil)
|
||||
|
||||
// NewConfigMapPayload constructs a Payload backed by a ConfigMap, which must have a non-empty UID
|
||||
func NewConfigMapPayload(cm *apiv1.ConfigMap) (Payload, error) {
|
||||
if cm == nil {
|
||||
return nil, fmt.Errorf("ConfigMap must be non-nil to be treated as a Checkpoint")
|
||||
} else if len(cm.ObjectMeta.UID) == 0 {
|
||||
return nil, fmt.Errorf("ConfigMap must have a UID to be treated as a Checkpoint")
|
||||
return nil, fmt.Errorf("ConfigMap must be non-nil")
|
||||
} else if cm.ObjectMeta.UID == "" {
|
||||
return nil, fmt.Errorf("ConfigMap must have a non-empty UID")
|
||||
} else if cm.ObjectMeta.ResourceVersion == "" {
|
||||
return nil, fmt.Errorf("ConfigMap must have a non-empty ResourceVersion")
|
||||
}
|
||||
|
||||
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &configMapCheckpoint{kubeletCodecs, cm}, nil
|
||||
return &configMapPayload{cm}, nil
|
||||
}
|
||||
|
||||
// UID returns the UID of a configMapCheckpoint
|
||||
func (c *configMapCheckpoint) UID() string {
|
||||
return string(c.configMap.UID)
|
||||
func (p *configMapPayload) UID() string {
|
||||
return string(p.cm.UID)
|
||||
}
|
||||
|
||||
// Parse extracts the KubeletConfiguration from v1/ConfigMap checkpoints, applies defaults, and converts to the internal type
|
||||
func (c *configMapCheckpoint) Parse() (*kubeletconfig.KubeletConfiguration, error) {
|
||||
const emptyCfgErr = "config was empty, but some parameters are required"
|
||||
|
||||
if len(c.configMap.Data) == 0 {
|
||||
return nil, fmt.Errorf(emptyCfgErr)
|
||||
}
|
||||
|
||||
config, ok := c.configMap.Data[configMapConfigKey]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("key %q not found in ConfigMap", configMapConfigKey)
|
||||
} else if len(config) == 0 {
|
||||
return nil, fmt.Errorf(emptyCfgErr)
|
||||
}
|
||||
|
||||
return utilcodec.DecodeKubeletConfiguration(c.kubeletCodecs, []byte(config))
|
||||
func (p *configMapPayload) ResourceVersion() string {
|
||||
return p.cm.ResourceVersion
|
||||
}
|
||||
|
||||
// Encode encodes a configMapCheckpoint
|
||||
func (c *configMapCheckpoint) Encode() ([]byte, error) {
|
||||
cm := c.configMap
|
||||
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := runtime.Encode(encoder, cm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
func (p *configMapPayload) Files() map[string]string {
|
||||
return p.cm.Data
|
||||
}
|
||||
|
||||
func (c *configMapCheckpoint) object() interface{} {
|
||||
return c.configMap
|
||||
func (p *configMapPayload) object() interface{} {
|
||||
return p.cm
|
||||
}
|
||||
|
269
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go
generated
vendored
269
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/configmap_test.go
generated
vendored
@ -17,7 +17,7 @@ limitations under the License.
|
||||
package checkpoint
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
@ -25,213 +25,124 @@ import (
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
|
||||
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
|
||||
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
|
||||
)
|
||||
|
||||
func TestNewConfigMapCheckpoint(t *testing.T) {
|
||||
func TestNewConfigMapPayload(t *testing.T) {
|
||||
cases := []struct {
|
||||
desc string
|
||||
cm *apiv1.ConfigMap
|
||||
err string
|
||||
}{
|
||||
{"nil v1/ConfigMap", nil, "must be non-nil"},
|
||||
{"empty v1/ConfigMap", &apiv1.ConfigMap{}, "must have a UID"},
|
||||
{"populated v1/ConfigMap",
|
||||
&apiv1.ConfigMap{
|
||||
{
|
||||
desc: "nil",
|
||||
cm: nil,
|
||||
err: "ConfigMap must be non-nil",
|
||||
},
|
||||
{
|
||||
desc: "missing uid",
|
||||
cm: &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "name",
|
||||
ResourceVersion: "rv",
|
||||
},
|
||||
},
|
||||
err: "ConfigMap must have a non-empty UID",
|
||||
},
|
||||
{
|
||||
desc: "missing resourceVersion",
|
||||
cm: &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "name",
|
||||
UID: types.UID("uid"),
|
||||
UID: "uid",
|
||||
},
|
||||
},
|
||||
err: "ConfigMap must have a non-empty ResourceVersion",
|
||||
},
|
||||
{
|
||||
desc: "populated v1/ConfigMap",
|
||||
cm: &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "name",
|
||||
UID: "uid",
|
||||
ResourceVersion: "rv",
|
||||
},
|
||||
Data: map[string]string{
|
||||
"key1": "value1",
|
||||
"key2": "value2",
|
||||
},
|
||||
}, ""},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
cpt, err := NewConfigMapCheckpoint(c.cm)
|
||||
if utiltest.SkipRest(t, c.desc, err, c.err) {
|
||||
continue
|
||||
}
|
||||
// underlying object should match the object passed in
|
||||
if !apiequality.Semantic.DeepEqual(cpt.object(), c.cm) {
|
||||
t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.cm), spew.Sdump(cpt))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMapCheckpointUID(t *testing.T) {
|
||||
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
|
||||
for _, uidIn := range cases {
|
||||
cpt := &configMapCheckpoint{
|
||||
kubeletCodecs,
|
||||
&apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{UID: types.UID(uidIn)},
|
||||
},
|
||||
}
|
||||
// UID method should return the correct value of the UID
|
||||
uidOut := cpt.UID()
|
||||
if uidIn != uidOut {
|
||||
t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
|
||||
}
|
||||
err: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
payload, err := NewConfigMapPayload(c.cm)
|
||||
utiltest.ExpectError(t, err, c.err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// underlying object should match the object passed in
|
||||
if !apiequality.Semantic.DeepEqual(c.cm, payload.object()) {
|
||||
t.Errorf("expect %s but got %s", spew.Sdump(c.cm), spew.Sdump(payload))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMapCheckpointParse(t *testing.T) {
|
||||
kubeletScheme, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
func TestConfigMapPayloadUID(t *testing.T) {
|
||||
const expect = "uid"
|
||||
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: expect, ResourceVersion: "rv"}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
t.Fatalf("error constructing payload: %v", err)
|
||||
}
|
||||
uid := payload.UID()
|
||||
if expect != uid {
|
||||
t.Errorf("expect %q, but got %q", expect, uid)
|
||||
}
|
||||
}
|
||||
|
||||
// get the built-in default configuration
|
||||
external := &kubeletconfigv1beta1.KubeletConfiguration{}
|
||||
kubeletScheme.Default(external)
|
||||
defaultConfig := &kubeletconfig.KubeletConfiguration{}
|
||||
err = kubeletScheme.Convert(external, defaultConfig, nil)
|
||||
func TestConfigMapPayloadResourceVersion(t *testing.T) {
|
||||
const expect = "rv"
|
||||
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: "uid", ResourceVersion: expect}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
t.Fatalf("error constructing payload: %v", err)
|
||||
}
|
||||
resourceVersion := payload.ResourceVersion()
|
||||
if expect != resourceVersion {
|
||||
t.Errorf("expect %q, but got %q", expect, resourceVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMapPayloadFiles(t *testing.T) {
|
||||
cases := []struct {
|
||||
desc string
|
||||
cm *apiv1.ConfigMap
|
||||
expect *kubeletconfig.KubeletConfiguration
|
||||
err string
|
||||
data map[string]string
|
||||
expect map[string]string
|
||||
}{
|
||||
{"empty data", &apiv1.ConfigMap{}, nil, "config was empty"},
|
||||
// missing kubelet key
|
||||
{"missing kubelet key", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"bogus": "stuff"}}, nil, fmt.Sprintf("key %q not found", configMapConfigKey)},
|
||||
// invalid format
|
||||
{"invalid yaml", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": "*"}}, nil, "failed to decode"},
|
||||
{"invalid json", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": "{*"}}, nil, "failed to decode"},
|
||||
// invalid object
|
||||
{"missing kind", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `{"apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, nil, "failed to decode"},
|
||||
{"missing version", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `{"kind":"KubeletConfiguration"}`}}, nil, "failed to decode"},
|
||||
{"unregistered kind", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `{"kind":"BogusKind","apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, nil, "failed to decode"},
|
||||
{"unregistered version", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"bogusversion"}`}}, nil, "failed to decode"},
|
||||
// empty object with correct kind and version should result in the defaults for that kind and version
|
||||
{"default from yaml", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `kind: KubeletConfiguration
|
||||
apiVersion: kubelet.config.k8s.io/v1beta1`}}, defaultConfig, ""},
|
||||
{"default from json", &apiv1.ConfigMap{Data: map[string]string{
|
||||
"kubelet": `{"kind":"KubeletConfiguration","apiVersion":"kubelet.config.k8s.io/v1beta1"}`}}, defaultConfig, ""},
|
||||
{"nil", nil, nil},
|
||||
{"empty", map[string]string{}, map[string]string{}},
|
||||
{"populated",
|
||||
map[string]string{
|
||||
"foo": "1",
|
||||
"bar": "2",
|
||||
},
|
||||
map[string]string{
|
||||
"foo": "1",
|
||||
"bar": "2",
|
||||
}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
cpt := &configMapCheckpoint{kubeletCodecs, c.cm}
|
||||
kc, err := cpt.Parse()
|
||||
if utiltest.SkipRest(t, c.desc, err, c.err) {
|
||||
continue
|
||||
}
|
||||
// we expect the parsed configuration to match what we described in the ConfigMap
|
||||
if !apiequality.Semantic.DeepEqual(c.expect, kc) {
|
||||
t.Errorf("case %q, expect config %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(kc))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMapCheckpointEncode(t *testing.T) {
|
||||
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// only one case, based on output from the existing encoder, and since
|
||||
// this is hard to test (key order isn't guaranteed), we should probably
|
||||
// just stick to this test case and mostly rely on the round-trip test.
|
||||
cases := []struct {
|
||||
desc string
|
||||
cpt *configMapCheckpoint
|
||||
expect string
|
||||
}{
|
||||
// we expect Checkpoints to be encoded as a json representation of the underlying API object
|
||||
{"one-key",
|
||||
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{Name: "one-key"},
|
||||
Data: map[string]string{"one": ""}}},
|
||||
`{"kind":"ConfigMap","apiVersion":"v1","metadata":{"name":"one-key","creationTimestamp":null},"data":{"one":""}}
|
||||
`},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
data, err := c.cpt.Encode()
|
||||
// we don't expect any errors from encoding
|
||||
if utiltest.SkipRest(t, c.desc, err, "") {
|
||||
continue
|
||||
}
|
||||
if string(data) != c.expect {
|
||||
t.Errorf("case %q, expect encoding %q but got %q", c.desc, c.expect, string(data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigMapCheckpointRoundTrip(t *testing.T) {
|
||||
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
desc string
|
||||
cpt *configMapCheckpoint
|
||||
decodeErr string
|
||||
}{
|
||||
// empty data
|
||||
{"empty data",
|
||||
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "empty-data-sha256-e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
UID: "uid",
|
||||
},
|
||||
Data: map[string]string{}}},
|
||||
""},
|
||||
// two keys
|
||||
{"two keys",
|
||||
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
|
||||
UID: "uid",
|
||||
},
|
||||
Data: map[string]string{"one": "", "two": "2"}}},
|
||||
""},
|
||||
// missing uid
|
||||
{"missing uid",
|
||||
&configMapCheckpoint{kubeletCodecs, &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "two-keys-sha256-2bff03d6249c8a9dc9a1436d087c124741361ccfac6615b81b67afcff5c42431",
|
||||
UID: "",
|
||||
},
|
||||
Data: map[string]string{"one": "", "two": "2"}}},
|
||||
"must have a UID"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
// we don't expect any errors from encoding
|
||||
data, err := c.cpt.Encode()
|
||||
if utiltest.SkipRest(t, c.desc, err, "") {
|
||||
continue
|
||||
}
|
||||
after, err := DecodeCheckpoint(data)
|
||||
if utiltest.SkipRest(t, c.desc, err, c.decodeErr) {
|
||||
continue
|
||||
}
|
||||
if !apiequality.Semantic.DeepEqual(c.cpt.object(), after.object()) {
|
||||
t.Errorf("case %q, expect round-trip result %s but got %s", c.desc, spew.Sdump(c.cpt), spew.Sdump(after))
|
||||
}
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
payload, err := NewConfigMapPayload(&apiv1.ConfigMap{ObjectMeta: metav1.ObjectMeta{UID: "uid", ResourceVersion: "rv"}, Data: c.data})
|
||||
if err != nil {
|
||||
t.Fatalf("error constructing payload: %v", err)
|
||||
}
|
||||
files := payload.Files()
|
||||
if !reflect.DeepEqual(c.expect, files) {
|
||||
t.Errorf("expected %v, but got %v", c.expect, files)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
254
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/download.go
generated
vendored
254
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/download.go
generated
vendored
@ -18,89 +18,130 @@ package checkpoint
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/fields"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
kuberuntime "k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/watch"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
kubeletconfiginternal "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
|
||||
kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/status"
|
||||
utilcodec "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/codec"
|
||||
utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
|
||||
)
|
||||
|
||||
// RemoteConfigSource represents a remote config source object that can be downloaded as a Checkpoint
|
||||
type RemoteConfigSource interface {
|
||||
// UID returns the UID of the remote config source object
|
||||
// Payload represents a local copy of a config source (payload) object
|
||||
type Payload interface {
|
||||
// UID returns a globally unique (space and time) identifier for the payload.
|
||||
// The return value is guaranteed non-empty.
|
||||
UID() string
|
||||
// APIPath returns the API path to the remote resource, e.g. its SelfLink
|
||||
APIPath() string
|
||||
// Download downloads the remote config source object returns a Checkpoint backed by the object,
|
||||
// or a sanitized failure reason and error if the download fails
|
||||
Download(client clientset.Interface) (Checkpoint, string, error)
|
||||
// Encode returns a []byte representation of the object behind the RemoteConfigSource
|
||||
Encode() ([]byte, error)
|
||||
|
||||
// object returns the underlying source object. If you want to compare sources for equality, use EqualRemoteConfigSources,
|
||||
// which compares the underlying source objects for semantic API equality.
|
||||
// ResourceVersion returns a resource version for the payload.
|
||||
// The return value is guaranteed non-empty.
|
||||
ResourceVersion() string
|
||||
|
||||
// Files returns a map of filenames to file contents.
|
||||
Files() map[string]string
|
||||
|
||||
// object returns the underlying checkpointed object.
|
||||
object() interface{}
|
||||
}
|
||||
|
||||
// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object, or returns
|
||||
// a sanitized failure reason and an error if the `source` is blatantly invalid.
|
||||
// RemoteConfigSource represents a remote config source object that can be downloaded as a Checkpoint
|
||||
type RemoteConfigSource interface {
|
||||
// KubeletFilename returns the name of the Kubelet config file as it should appear in the keys of Payload.Files()
|
||||
KubeletFilename() string
|
||||
|
||||
// APIPath returns the API path to the remote resource, e.g. its SelfLink
|
||||
APIPath() string
|
||||
|
||||
// UID returns the globally unique identifier for the most recently downloaded payload targeted by the source.
|
||||
UID() string
|
||||
|
||||
// ResourceVersion returns the resource version of the most recently downloaded payload targeted by the source.
|
||||
ResourceVersion() string
|
||||
|
||||
// Download downloads the remote config source's target object and returns a Payload backed by the object,
|
||||
// or a sanitized failure reason and error if the download fails.
|
||||
// Download takes an optional store as an argument. If provided, Download will check this store for the
|
||||
// target object prior to contacting the API server.
|
||||
// Download updates the local UID and ResourceVersion tracked by this source, based on the downloaded payload.
|
||||
Download(client clientset.Interface, store cache.Store) (Payload, string, error)
|
||||
|
||||
// Informer returns an informer that can be used to detect changes to the remote config source
|
||||
Informer(client clientset.Interface, handler cache.ResourceEventHandlerFuncs) cache.SharedInformer
|
||||
|
||||
// Encode returns a []byte representation of the object behind the RemoteConfigSource
|
||||
Encode() ([]byte, error)
|
||||
|
||||
// NodeConfigSource returns a copy of the underlying apiv1.NodeConfigSource object.
|
||||
// All RemoteConfigSources are expected to be backed by a NodeConfigSource,
|
||||
// though the convenience methods on the interface will target the source
|
||||
// type that was detected in a call to NewRemoteConfigSource.
|
||||
NodeConfigSource() *apiv1.NodeConfigSource
|
||||
}
|
||||
|
||||
// NewRemoteConfigSource constructs a RemoteConfigSource from a v1/NodeConfigSource object
|
||||
// You should only call this with a non-nil config source.
|
||||
// Note that the API server validates Node.Spec.ConfigSource.
|
||||
func NewRemoteConfigSource(source *apiv1.NodeConfigSource) (RemoteConfigSource, string, error) {
|
||||
// exactly one subfield of the config source must be non-nil, toady ConfigMapRef is the only reference
|
||||
if source.ConfigMapRef == nil {
|
||||
return nil, status.FailSyncReasonAllNilSubfields, fmt.Errorf("%s, NodeConfigSource was: %#v", status.FailSyncReasonAllNilSubfields, source)
|
||||
// NOTE: Even though the API server validates the config, we check whether all *known* fields are
|
||||
// nil here, so that if a new API server allows a new config source type, old clients can send
|
||||
// an error message rather than crashing due to a nil pointer dereference.
|
||||
|
||||
// Exactly one reference subfield of the config source must be non-nil.
|
||||
// Currently ConfigMap is the only reference subfield.
|
||||
if source.ConfigMap == nil {
|
||||
return nil, status.AllNilSubfieldsError, fmt.Errorf("%s, NodeConfigSource was: %#v", status.AllNilSubfieldsError, source)
|
||||
}
|
||||
|
||||
// validate the NodeConfigSource:
|
||||
|
||||
// at this point we know we're using the ConfigMapRef subfield
|
||||
ref := source.ConfigMapRef
|
||||
|
||||
// name, namespace, and UID must all be non-empty for ConfigMapRef
|
||||
if ref.Name == "" || ref.Namespace == "" || string(ref.UID) == "" {
|
||||
return nil, status.FailSyncReasonPartialObjectReference, fmt.Errorf("%s, ObjectReference was: %#v", status.FailSyncReasonPartialObjectReference, ref)
|
||||
}
|
||||
|
||||
return &remoteConfigMap{source}, "", nil
|
||||
}
|
||||
|
||||
// DecodeRemoteConfigSource is a helper for using the apimachinery to decode serialized RemoteConfigSources;
|
||||
// e.g. the objects stored in the .cur and .lkg files by checkpoint/store/fsstore.go
|
||||
// e.g. the metadata stored by checkpoint/store/fsstore.go
|
||||
func DecodeRemoteConfigSource(data []byte) (RemoteConfigSource, error) {
|
||||
// decode the remote config source
|
||||
obj, err := runtime.Decode(legacyscheme.Codecs.UniversalDecoder(), data)
|
||||
_, codecs, err := scheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
obj, err := runtime.Decode(codecs.UniversalDecoder(), data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode, error: %v", err)
|
||||
}
|
||||
|
||||
// for now we assume we are trying to load an apiv1.NodeConfigSource,
|
||||
// for now we assume we are trying to load an kubeletconfigv1beta1.SerializedNodeConfigSource,
|
||||
// this may need to be extended if e.g. a new version of the api is born
|
||||
|
||||
// convert it to the external NodeConfigSource type, so we're consistently working with the external type outside of the on-disk representation
|
||||
cs := &apiv1.NodeConfigSource{}
|
||||
err = legacyscheme.Scheme.Convert(obj, cs, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to convert decoded object into a v1 NodeConfigSource, error: %v", err)
|
||||
cs, ok := obj.(*kubeletconfiginternal.SerializedNodeConfigSource)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("failed to cast decoded remote config source to *k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig.SerializedNodeConfigSource")
|
||||
}
|
||||
source, _, err := NewRemoteConfigSource(cs)
|
||||
return source, err
|
||||
|
||||
// we use the v1.NodeConfigSource type on internal and external, so no need to convert to external here
|
||||
source, _, err := NewRemoteConfigSource(&cs.Source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return source, nil
|
||||
}
|
||||
|
||||
// EqualRemoteConfigSources is a helper for comparing remote config sources by
|
||||
// comparing the underlying API objects for semantic equality.
|
||||
func EqualRemoteConfigSources(a, b RemoteConfigSource) bool {
|
||||
if a != nil && b != nil {
|
||||
return apiequality.Semantic.DeepEqual(a.object(), b.object())
|
||||
return apiequality.Semantic.DeepEqual(a.NodeConfigSource(), b.NodeConfigSource())
|
||||
}
|
||||
if a == nil && b == nil {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
return a == b
|
||||
}
|
||||
|
||||
// remoteConfigMap implements RemoteConfigSource for v1/ConfigMap config sources
|
||||
@ -108,58 +149,125 @@ type remoteConfigMap struct {
|
||||
source *apiv1.NodeConfigSource
|
||||
}
|
||||
|
||||
func (r *remoteConfigMap) UID() string {
|
||||
return string(r.source.ConfigMapRef.UID)
|
||||
var _ RemoteConfigSource = (*remoteConfigMap)(nil)
|
||||
|
||||
func (r *remoteConfigMap) KubeletFilename() string {
|
||||
return r.source.ConfigMap.KubeletConfigKey
|
||||
}
|
||||
|
||||
const configMapAPIPathFmt = "/api/v1/namespaces/%s/configmaps/%s"
|
||||
|
||||
func (r *remoteConfigMap) APIPath() string {
|
||||
ref := r.source.ConfigMapRef
|
||||
ref := r.source.ConfigMap
|
||||
return fmt.Sprintf(configMapAPIPathFmt, ref.Namespace, ref.Name)
|
||||
}
|
||||
|
||||
func (r *remoteConfigMap) Download(client clientset.Interface) (Checkpoint, string, error) {
|
||||
var reason string
|
||||
uid := string(r.source.ConfigMapRef.UID)
|
||||
func (r *remoteConfigMap) UID() string {
|
||||
return string(r.source.ConfigMap.UID)
|
||||
}
|
||||
|
||||
utillog.Infof("attempting to download ConfigMap with UID %q", uid)
|
||||
func (r *remoteConfigMap) ResourceVersion() string {
|
||||
return r.source.ConfigMap.ResourceVersion
|
||||
}
|
||||
|
||||
// get the ConfigMap via namespace/name, there doesn't seem to be a way to get it by UID
|
||||
cm, err := client.CoreV1().ConfigMaps(r.source.ConfigMapRef.Namespace).Get(r.source.ConfigMapRef.Name, metav1.GetOptions{})
|
||||
func (r *remoteConfigMap) Download(client clientset.Interface, store cache.Store) (Payload, string, error) {
|
||||
var (
|
||||
cm *apiv1.ConfigMap
|
||||
err error
|
||||
)
|
||||
// check the in-memory store for the ConfigMap, so we can skip unnecessary downloads
|
||||
if store != nil {
|
||||
utillog.Infof("checking in-memory store for %s", r.APIPath())
|
||||
cm, err = getConfigMapFromStore(store, r.source.ConfigMap.Namespace, r.source.ConfigMap.Name)
|
||||
if err != nil {
|
||||
// just log the error, we'll attempt a direct download instead
|
||||
utillog.Errorf("failed to check in-memory store for %s, error: %v", r.APIPath(), err)
|
||||
} else if cm != nil {
|
||||
utillog.Infof("found %s in in-memory store, UID: %s, ResourceVersion: %s", r.APIPath(), cm.UID, cm.ResourceVersion)
|
||||
} else {
|
||||
utillog.Infof("did not find %s in in-memory store", r.APIPath())
|
||||
}
|
||||
}
|
||||
// if we didn't find the ConfigMap in the in-memory store, download it from the API server
|
||||
if cm == nil {
|
||||
utillog.Infof("attempting to download %s", r.APIPath())
|
||||
cm, err = client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Get(r.source.ConfigMap.Name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return nil, status.DownloadError, fmt.Errorf("%s, error: %v", status.DownloadError, err)
|
||||
}
|
||||
utillog.Infof("successfully downloaded %s, UID: %s, ResourceVersion: %s", r.APIPath(), cm.UID, cm.ResourceVersion)
|
||||
} // Assert: Now we have a non-nil ConfigMap
|
||||
// construct Payload from the ConfigMap
|
||||
payload, err := NewConfigMapPayload(cm)
|
||||
if err != nil {
|
||||
reason = fmt.Sprintf(status.FailSyncReasonDownloadFmt, r.APIPath())
|
||||
return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
|
||||
// We only expect an error here if ObjectMeta is lacking UID or ResourceVersion. This should
|
||||
// never happen on objects in the informer's store, or objects downloaded from the API server
|
||||
// directly, so we report InternalError.
|
||||
return nil, status.InternalError, fmt.Errorf("%s, error: %v", status.InternalError, err)
|
||||
}
|
||||
// update internal UID and ResourceVersion based on latest ConfigMap
|
||||
r.source.ConfigMap.UID = cm.UID
|
||||
r.source.ConfigMap.ResourceVersion = cm.ResourceVersion
|
||||
return payload, "", nil
|
||||
}
|
||||
|
||||
func (r *remoteConfigMap) Informer(client clientset.Interface, handler cache.ResourceEventHandlerFuncs) cache.SharedInformer {
|
||||
// select ConfigMap by name
|
||||
fieldselector := fields.OneTermEqualSelector("metadata.name", r.source.ConfigMap.Name)
|
||||
|
||||
// add some randomness to resync period, which can help avoid controllers falling into lock-step
|
||||
minResyncPeriod := 15 * time.Minute
|
||||
factor := rand.Float64() + 1
|
||||
resyncPeriod := time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor)
|
||||
|
||||
lw := &cache.ListWatch{
|
||||
ListFunc: func(options metav1.ListOptions) (kuberuntime.Object, error) {
|
||||
return client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).List(metav1.ListOptions{
|
||||
FieldSelector: fieldselector.String(),
|
||||
})
|
||||
},
|
||||
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
|
||||
return client.CoreV1().ConfigMaps(r.source.ConfigMap.Namespace).Watch(metav1.ListOptions{
|
||||
FieldSelector: fieldselector.String(),
|
||||
ResourceVersion: options.ResourceVersion,
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
// ensure that UID matches the UID on the reference, the ObjectReference must be unambiguous
|
||||
if r.source.ConfigMapRef.UID != cm.UID {
|
||||
reason = fmt.Sprintf(status.FailSyncReasonUIDMismatchFmt, r.source.ConfigMapRef.UID, r.APIPath(), cm.UID)
|
||||
return nil, reason, fmt.Errorf(reason)
|
||||
}
|
||||
informer := cache.NewSharedInformer(lw, &apiv1.ConfigMap{}, resyncPeriod)
|
||||
informer.AddEventHandler(handler)
|
||||
|
||||
checkpoint, err := NewConfigMapCheckpoint(cm)
|
||||
if err != nil {
|
||||
reason = fmt.Sprintf("invalid downloaded object")
|
||||
return nil, reason, fmt.Errorf("%s, error: %v", reason, err)
|
||||
}
|
||||
|
||||
utillog.Infof("successfully downloaded ConfigMap with UID %q", uid)
|
||||
return checkpoint, "", nil
|
||||
return informer
|
||||
}
|
||||
|
||||
func (r *remoteConfigMap) Encode() ([]byte, error) {
|
||||
encoder, err := utilcodec.NewJSONEncoder(apiv1.GroupName)
|
||||
encoder, err := utilcodec.NewKubeletconfigYAMLEncoder(kubeletconfigv1beta1.SchemeGroupVersion)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := runtime.Encode(encoder, r.source)
|
||||
|
||||
data, err := runtime.Encode(encoder, &kubeletconfigv1beta1.SerializedNodeConfigSource{Source: *r.source})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (r *remoteConfigMap) object() interface{} {
|
||||
return r.source
|
||||
func (r *remoteConfigMap) NodeConfigSource() *apiv1.NodeConfigSource {
|
||||
return r.source.DeepCopy()
|
||||
}
|
||||
|
||||
func getConfigMapFromStore(store cache.Store, namespace, name string) (*apiv1.ConfigMap, error) {
|
||||
key := fmt.Sprintf("%s/%s", namespace, name)
|
||||
obj, ok, err := store.GetByKey(key)
|
||||
if err != nil || !ok {
|
||||
return nil, err
|
||||
}
|
||||
cm, ok := obj.(*apiv1.ConfigMap)
|
||||
if !ok {
|
||||
err := fmt.Errorf("failed to cast object %s from informer's store to ConfigMap", key)
|
||||
utillog.Errorf(err.Error())
|
||||
return nil, err
|
||||
}
|
||||
return cm, nil
|
||||
}
|
||||
|
261
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/download_test.go
generated
vendored
261
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/download_test.go
generated
vendored
@ -25,9 +25,9 @@ import (
|
||||
apiv1 "k8s.io/api/core/v1"
|
||||
apiequality "k8s.io/apimachinery/pkg/api/equality"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/types"
|
||||
clientset "k8s.io/client-go/kubernetes"
|
||||
fakeclient "k8s.io/client-go/kubernetes/fake"
|
||||
kubeletscheme "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/scheme"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
utiltest "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/test"
|
||||
)
|
||||
|
||||
@ -38,118 +38,205 @@ func TestNewRemoteConfigSource(t *testing.T) {
|
||||
expect RemoteConfigSource
|
||||
err string
|
||||
}{
|
||||
// all NodeConfigSource subfields nil
|
||||
{"all NodeConfigSource subfields nil",
|
||||
&apiv1.NodeConfigSource{}, nil, "exactly one subfield must be non-nil"},
|
||||
{"ConfigMapRef: empty name, namespace, and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty name and namespace
|
||||
{"ConfigMapRef: empty name and namespace",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{UID: "uid"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty name and UID
|
||||
{"ConfigMapRef: empty name and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty namespace and UID
|
||||
{"ConfigMapRef: empty namespace and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty UID
|
||||
{"ConfigMapRef: empty namespace and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty namespace
|
||||
{"ConfigMapRef: empty namespace and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", UID: "uid"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: empty name
|
||||
{"ConfigMapRef: empty namespace and UID",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Namespace: "namespace", UID: "uid"}}, nil, "invalid ObjectReference"},
|
||||
// ConfigMapRef: valid reference
|
||||
{"ConfigMapRef: valid reference",
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}},
|
||||
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}}, ""},
|
||||
{
|
||||
desc: "all NodeConfigSource subfields nil",
|
||||
source: &apiv1.NodeConfigSource{},
|
||||
expect: nil,
|
||||
err: "exactly one subfield must be non-nil",
|
||||
},
|
||||
{
|
||||
desc: "ConfigMap: valid reference",
|
||||
source: &apiv1.NodeConfigSource{
|
||||
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: "uid",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}},
|
||||
expect: &remoteConfigMap{&apiv1.NodeConfigSource{
|
||||
ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: "uid",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}}},
|
||||
err: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
src, _, err := NewRemoteConfigSource(c.source)
|
||||
if utiltest.SkipRest(t, c.desc, err, c.err) {
|
||||
continue
|
||||
}
|
||||
// underlying object should match the object passed in
|
||||
if !apiequality.Semantic.DeepEqual(c.expect.object(), src.object()) {
|
||||
t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(src))
|
||||
}
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
source, _, err := NewRemoteConfigSource(c.source)
|
||||
utiltest.ExpectError(t, err, c.err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// underlying object should match the object passed in
|
||||
if !apiequality.Semantic.DeepEqual(c.expect.NodeConfigSource(), source.NodeConfigSource()) {
|
||||
t.Errorf("case %q, expect RemoteConfigSource %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(source))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteConfigMapUID(t *testing.T) {
|
||||
cases := []string{"", "uid", "376dfb73-56db-11e7-a01e-42010a800002"}
|
||||
for _, uidIn := range cases {
|
||||
cpt := &remoteConfigMap{
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: types.UID(uidIn)}},
|
||||
}
|
||||
// UID method should return the correct value of the UID
|
||||
uidOut := cpt.UID()
|
||||
if uidIn != uidOut {
|
||||
t.Errorf("expect UID() to return %q, but got %q", uidIn, uidOut)
|
||||
}
|
||||
const expect = "uid"
|
||||
source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: expect,
|
||||
KubeletConfigKey: "kubelet",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("error constructing remote config source: %v", err)
|
||||
}
|
||||
uid := source.UID()
|
||||
if expect != uid {
|
||||
t.Errorf("expect %q, but got %q", expect, uid)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteConfigMapAPIPath(t *testing.T) {
|
||||
name := "name"
|
||||
namespace := "namespace"
|
||||
cpt := &remoteConfigMap{
|
||||
&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: name, Namespace: namespace, UID: ""}},
|
||||
const (
|
||||
name = "name"
|
||||
namespace = "namespace"
|
||||
)
|
||||
source, _, err := NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: name,
|
||||
Namespace: namespace,
|
||||
UID: "uid",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("error constructing remote config source: %v", err)
|
||||
}
|
||||
expect := fmt.Sprintf(configMapAPIPathFmt, cpt.source.ConfigMapRef.Namespace, cpt.source.ConfigMapRef.Name)
|
||||
// APIPath() method should return the correct path to the referenced resource
|
||||
path := cpt.APIPath()
|
||||
expect := fmt.Sprintf(configMapAPIPathFmt, namespace, name)
|
||||
path := source.APIPath()
|
||||
|
||||
if expect != path {
|
||||
t.Errorf("expect APIPath() to return %q, but got %q", expect, namespace)
|
||||
t.Errorf("expect %q, but got %q", expect, path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoteConfigMapDownload(t *testing.T) {
|
||||
_, kubeletCodecs, err := kubeletscheme.NewSchemeAndCodecs()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
cm := &apiv1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: "uid",
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: "uid",
|
||||
ResourceVersion: "1",
|
||||
}}
|
||||
client := fakeclient.NewSimpleClientset(cm)
|
||||
|
||||
source := &apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}}
|
||||
|
||||
expectPayload, err := NewConfigMapPayload(cm)
|
||||
if err != nil {
|
||||
t.Fatalf("error constructing payload: %v", err)
|
||||
}
|
||||
|
||||
missingStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
|
||||
hasStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
|
||||
if err := hasStore.Add(cm); err != nil {
|
||||
t.Fatalf("unexpected error constructing hasStore")
|
||||
}
|
||||
|
||||
missingClient := fakeclient.NewSimpleClientset()
|
||||
hasClient := fakeclient.NewSimpleClientset(cm)
|
||||
|
||||
cases := []struct {
|
||||
desc string
|
||||
source RemoteConfigSource
|
||||
expect Checkpoint
|
||||
client clientset.Interface
|
||||
store cache.Store
|
||||
err string
|
||||
}{
|
||||
|
||||
// object doesn't exist
|
||||
{"object doesn't exist",
|
||||
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "bogus", Namespace: "namespace", UID: "bogus"}}},
|
||||
nil, "not found"},
|
||||
// UID of downloaded object doesn't match UID of referent found via namespace/name
|
||||
{"UID is incorrect for namespace/name",
|
||||
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "bogus"}}},
|
||||
nil, "does not match"},
|
||||
// successful download
|
||||
{"object exists and reference is correct",
|
||||
&remoteConfigMap{&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}}},
|
||||
&configMapCheckpoint{kubeletCodecs, cm}, ""},
|
||||
{
|
||||
desc: "nil store, object does not exist in API server",
|
||||
client: missingClient,
|
||||
err: "not found",
|
||||
},
|
||||
{
|
||||
desc: "nil store, object exists in API server",
|
||||
client: hasClient,
|
||||
},
|
||||
{
|
||||
desc: "object exists in store and API server",
|
||||
store: hasStore,
|
||||
client: hasClient,
|
||||
},
|
||||
{
|
||||
desc: "object exists in store, but does not exist in API server",
|
||||
store: hasStore,
|
||||
client: missingClient,
|
||||
},
|
||||
{
|
||||
desc: "object does not exist in store, but exists in API server",
|
||||
store: missingStore,
|
||||
client: hasClient,
|
||||
},
|
||||
{
|
||||
desc: "object does not exist in store or API server",
|
||||
client: missingClient,
|
||||
store: missingStore,
|
||||
err: "not found",
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
cpt, _, err := c.source.Download(client)
|
||||
if utiltest.SkipRest(t, c.desc, err, c.err) {
|
||||
continue
|
||||
}
|
||||
// "downloaded" object should match the expected
|
||||
if !apiequality.Semantic.DeepEqual(c.expect.object(), cpt.object()) {
|
||||
t.Errorf("case %q, expect Checkpoint %s but got %s", c.desc, spew.Sdump(c.expect), spew.Sdump(cpt))
|
||||
}
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
// deep copy so we can always check the UID/ResourceVersion are set after Download
|
||||
s, _, err := NewRemoteConfigSource(source.DeepCopy())
|
||||
if err != nil {
|
||||
t.Fatalf("error constructing remote config source %v", err)
|
||||
}
|
||||
// attempt download
|
||||
p, _, err := s.Download(c.client, c.store)
|
||||
utiltest.ExpectError(t, err, c.err)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// downloaded object should match the expected
|
||||
if !apiequality.Semantic.DeepEqual(expectPayload.object(), p.object()) {
|
||||
t.Errorf("expect Checkpoint %s but got %s", spew.Sdump(expectPayload), spew.Sdump(p))
|
||||
}
|
||||
// source UID and ResourceVersion should be updated by Download
|
||||
if p.UID() != s.UID() {
|
||||
t.Errorf("expect UID to be updated by Download to match payload: %s, but got source UID: %s", p.UID(), s.UID())
|
||||
}
|
||||
if p.ResourceVersion() != s.ResourceVersion() {
|
||||
t.Errorf("expect ResourceVersion to be updated by Download to match payload: %s, but got source ResourceVersion: %s", p.ResourceVersion(), s.ResourceVersion())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEqualRemoteConfigSources(t *testing.T) {
|
||||
cases := []struct {
|
||||
desc string
|
||||
a RemoteConfigSource
|
||||
b RemoteConfigSource
|
||||
expect bool
|
||||
}{
|
||||
{"both nil", nil, nil, true},
|
||||
{"a nil", nil, &remoteConfigMap{}, false},
|
||||
{"b nil", &remoteConfigMap{}, nil, false},
|
||||
{"neither nil, equal", &remoteConfigMap{}, &remoteConfigMap{}, true},
|
||||
{
|
||||
desc: "neither nil, not equal",
|
||||
a: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{Name: "a"}}},
|
||||
b: &remoteConfigMap{&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{KubeletConfigKey: "kubelet"}}},
|
||||
expect: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.desc, func(t *testing.T) {
|
||||
if EqualRemoteConfigSources(c.a, c.b) != c.expect {
|
||||
t.Errorf("expected EqualRemoteConfigSources to return %t, but got %t", c.expect, !c.expect)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
6
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/BUILD
generated
vendored
6
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/BUILD
generated
vendored
@ -14,7 +14,11 @@ go_test(
|
||||
],
|
||||
embed = [":go_default_library"],
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/scheme:go_default_library",
|
||||
"//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/codec:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/files:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/test:go_default_library",
|
||||
"//pkg/util/filesystem:go_default_library",
|
||||
@ -34,7 +38,9 @@ go_library(
|
||||
],
|
||||
importpath = "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store",
|
||||
deps = [
|
||||
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/checkpoint:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/configfiles:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/files:go_default_library",
|
||||
"//pkg/kubelet/kubeletconfig/util/log:go_default_library",
|
||||
"//pkg/util/filesystem:go_default_library",
|
||||
|
29
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go
generated
vendored
29
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fakestore.go
generated
vendored
@ -20,52 +20,51 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
|
||||
)
|
||||
|
||||
// so far only implements Current(), LastKnownGood(), SetCurrent(), and SetLastKnownGood()
|
||||
// so far only implements Assigned(), LastKnownGood(), SetAssigned(), and SetLastKnownGood()
|
||||
type fakeStore struct {
|
||||
current checkpoint.RemoteConfigSource
|
||||
assigned checkpoint.RemoteConfigSource
|
||||
lastKnownGood checkpoint.RemoteConfigSource
|
||||
}
|
||||
|
||||
var _ Store = (*fakeStore)(nil)
|
||||
|
||||
func (s *fakeStore) Initialize() error {
|
||||
return fmt.Errorf("Initialize method not supported")
|
||||
}
|
||||
|
||||
func (s *fakeStore) Exists(uid string) (bool, error) {
|
||||
func (s *fakeStore) Exists(source checkpoint.RemoteConfigSource) (bool, error) {
|
||||
return false, fmt.Errorf("Exists method not supported")
|
||||
}
|
||||
|
||||
func (s *fakeStore) Save(c checkpoint.Checkpoint) error {
|
||||
func (s *fakeStore) Save(c checkpoint.Payload) error {
|
||||
return fmt.Errorf("Save method not supported")
|
||||
}
|
||||
|
||||
func (s *fakeStore) Load(uid string) (checkpoint.Checkpoint, error) {
|
||||
func (s *fakeStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error) {
|
||||
return nil, fmt.Errorf("Load method not supported")
|
||||
}
|
||||
|
||||
func (s *fakeStore) CurrentModified() (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("CurrentModified method not supported")
|
||||
func (s *fakeStore) AssignedModified() (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("AssignedModified method not supported")
|
||||
}
|
||||
|
||||
func (s *fakeStore) Current() (checkpoint.RemoteConfigSource, error) {
|
||||
return s.current, nil
|
||||
func (s *fakeStore) Assigned() (checkpoint.RemoteConfigSource, error) {
|
||||
return s.assigned, nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
|
||||
return s.lastKnownGood, nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
|
||||
s.current = source
|
||||
func (s *fakeStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
|
||||
s.assigned = source
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *fakeStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
|
||||
return setCurrentUpdated(s, source)
|
||||
}
|
||||
|
||||
func (s *fakeStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
|
||||
s.lastKnownGood = source
|
||||
return nil
|
||||
|
149
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go
generated
vendored
149
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore.go
generated
vendored
@ -21,82 +21,112 @@ import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles"
|
||||
utilfiles "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/files"
|
||||
utillog "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/util/log"
|
||||
utilfs "k8s.io/kubernetes/pkg/util/filesystem"
|
||||
)
|
||||
|
||||
const (
|
||||
curFile = ".cur"
|
||||
lkgFile = ".lkg"
|
||||
metaDir = "meta"
|
||||
assignedFile = "assigned"
|
||||
lastKnownGoodFile = "last-known-good"
|
||||
|
||||
checkpointsDir = "checkpoints"
|
||||
)
|
||||
|
||||
// fsStore is for tracking checkpoints in the local filesystem, implements Store
|
||||
type fsStore struct {
|
||||
// fs is the filesystem to use for storage operations; can be mocked for testing
|
||||
fs utilfs.Filesystem
|
||||
// checkpointsDir is the absolute path to the storage directory for fsStore
|
||||
checkpointsDir string
|
||||
// dir is the absolute path to the storage directory for fsStore
|
||||
dir string
|
||||
}
|
||||
|
||||
// NewFsStore returns a Store that saves its data in `checkpointsDir`
|
||||
func NewFsStore(fs utilfs.Filesystem, checkpointsDir string) Store {
|
||||
var _ Store = (*fsStore)(nil)
|
||||
|
||||
// NewFsStore returns a Store that saves its data in dir
|
||||
func NewFsStore(fs utilfs.Filesystem, dir string) Store {
|
||||
return &fsStore{
|
||||
fs: fs,
|
||||
checkpointsDir: checkpointsDir,
|
||||
fs: fs,
|
||||
dir: dir,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *fsStore) Initialize() error {
|
||||
utillog.Infof("initializing config checkpoints directory %q", s.checkpointsDir)
|
||||
if err := utilfiles.EnsureDir(s.fs, s.checkpointsDir); err != nil {
|
||||
utillog.Infof("initializing config checkpoints directory %q", s.dir)
|
||||
// ensure top-level dir for store
|
||||
if err := utilfiles.EnsureDir(s.fs, s.dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, curFile)); err != nil {
|
||||
// ensure metadata directory and reference files (tracks assigned and lkg configs)
|
||||
if err := utilfiles.EnsureDir(s.fs, filepath.Join(s.dir, metaDir)); err != nil {
|
||||
return err
|
||||
}
|
||||
return utilfiles.EnsureFile(s.fs, filepath.Join(s.checkpointsDir, lkgFile))
|
||||
if err := utilfiles.EnsureFile(s.fs, s.metaPath(assignedFile)); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := utilfiles.EnsureFile(s.fs, s.metaPath(lastKnownGoodFile)); err != nil {
|
||||
return err
|
||||
}
|
||||
// ensure checkpoints directory (saves unpacked payloads in subdirectories named after payload UID)
|
||||
return utilfiles.EnsureDir(s.fs, filepath.Join(s.dir, checkpointsDir))
|
||||
}
|
||||
|
||||
func (s *fsStore) Exists(uid string) (bool, error) {
|
||||
ok, err := utilfiles.FileExists(s.fs, filepath.Join(s.checkpointsDir, uid))
|
||||
func (s *fsStore) Exists(source checkpoint.RemoteConfigSource) (bool, error) {
|
||||
const errfmt = "failed to determine whether checkpoint exists for source %s, UID: %s, ResourceVersion: %s exists, error: %v"
|
||||
if len(source.UID()) == 0 {
|
||||
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), "empty UID is ambiguous")
|
||||
}
|
||||
if len(source.ResourceVersion()) == 0 {
|
||||
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), "empty ResourceVersion is ambiguous")
|
||||
}
|
||||
|
||||
// we check whether the directory was created for the resource
|
||||
ok, err := utilfiles.DirExists(s.fs, s.checkpointPath(source.UID(), source.ResourceVersion()))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to determine whether checkpoint %q exists, error: %v", uid, err)
|
||||
return false, fmt.Errorf(errfmt, source.APIPath(), source.UID(), source.ResourceVersion(), err)
|
||||
}
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func (s *fsStore) Save(c checkpoint.Checkpoint) error {
|
||||
// encode the checkpoint
|
||||
data, err := c.Encode()
|
||||
if err != nil {
|
||||
func (s *fsStore) Save(payload checkpoint.Payload) error {
|
||||
// Note: Payload interface guarantees UID() and ResourceVersion() to be non-empty
|
||||
path := s.checkpointPath(payload.UID(), payload.ResourceVersion())
|
||||
// ensure the parent dir (checkpoints/uid) exists, since ReplaceDir requires the parent of the replacee
|
||||
// to exist, and we checkpoint as checkpoints/uid/resourceVersion/files-from-configmap
|
||||
if err := utilfiles.EnsureDir(s.fs, filepath.Dir(path)); err != nil {
|
||||
return err
|
||||
}
|
||||
// save the file
|
||||
return utilfiles.ReplaceFile(s.fs, filepath.Join(s.checkpointsDir, c.UID()), data)
|
||||
// save the checkpoint's files in the appropriate checkpoint dir
|
||||
return utilfiles.ReplaceDir(s.fs, path, payload.Files())
|
||||
}
|
||||
|
||||
func (s *fsStore) Load(uid string) (checkpoint.Checkpoint, error) {
|
||||
filePath := filepath.Join(s.checkpointsDir, uid)
|
||||
utillog.Infof("loading configuration from %q", filePath)
|
||||
|
||||
// load the file
|
||||
data, err := s.fs.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read checkpoint file %q, error: %v", filePath, err)
|
||||
func (s *fsStore) Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error) {
|
||||
sourceFmt := fmt.Sprintf("%s, UID: %s, ResourceVersion: %s", source.APIPath(), source.UID(), source.ResourceVersion())
|
||||
// check if a checkpoint exists for the source
|
||||
if ok, err := s.Exists(source); err != nil {
|
||||
return nil, err
|
||||
} else if !ok {
|
||||
return nil, fmt.Errorf("no checkpoint for source %s", sourceFmt)
|
||||
}
|
||||
|
||||
// decode it
|
||||
c, err := checkpoint.DecodeCheckpoint(data)
|
||||
// load the kubelet config file
|
||||
utillog.Infof("loading Kubelet configuration checkpoint for source %s", sourceFmt)
|
||||
loader, err := configfiles.NewFsLoader(s.fs, filepath.Join(s.checkpointPath(source.UID(), source.ResourceVersion()), source.KubeletFilename()))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to decode checkpoint file %q, error: %v", filePath, err)
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
kc, err := loader.Load()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return kc, nil
|
||||
}
|
||||
|
||||
func (s *fsStore) CurrentModified() (time.Time, error) {
|
||||
path := filepath.Join(s.checkpointsDir, curFile)
|
||||
func (s *fsStore) AssignedModified() (time.Time, error) {
|
||||
path := s.metaPath(assignedFile)
|
||||
info, err := s.fs.Stat(path)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("failed to stat %q while checking modification time, error: %v", path, err)
|
||||
@ -104,35 +134,36 @@ func (s *fsStore) CurrentModified() (time.Time, error) {
|
||||
return info.ModTime(), nil
|
||||
}
|
||||
|
||||
func (s *fsStore) Current() (checkpoint.RemoteConfigSource, error) {
|
||||
return s.sourceFromFile(curFile)
|
||||
func (s *fsStore) Assigned() (checkpoint.RemoteConfigSource, error) {
|
||||
return readRemoteConfigSource(s.fs, s.metaPath(assignedFile))
|
||||
}
|
||||
|
||||
func (s *fsStore) LastKnownGood() (checkpoint.RemoteConfigSource, error) {
|
||||
return s.sourceFromFile(lkgFile)
|
||||
return readRemoteConfigSource(s.fs, s.metaPath(lastKnownGoodFile))
|
||||
}
|
||||
|
||||
func (s *fsStore) SetCurrent(source checkpoint.RemoteConfigSource) error {
|
||||
return s.setSourceFile(curFile, source)
|
||||
}
|
||||
|
||||
func (s *fsStore) SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error) {
|
||||
return setCurrentUpdated(s, source)
|
||||
func (s *fsStore) SetAssigned(source checkpoint.RemoteConfigSource) error {
|
||||
return writeRemoteConfigSource(s.fs, s.metaPath(assignedFile), source)
|
||||
}
|
||||
|
||||
func (s *fsStore) SetLastKnownGood(source checkpoint.RemoteConfigSource) error {
|
||||
return s.setSourceFile(lkgFile, source)
|
||||
return writeRemoteConfigSource(s.fs, s.metaPath(lastKnownGoodFile), source)
|
||||
}
|
||||
|
||||
func (s *fsStore) Reset() (bool, error) {
|
||||
return reset(s)
|
||||
}
|
||||
|
||||
// sourceFromFile returns the RemoteConfigSource stored in the file at `s.checkpointsDir/relPath`,
|
||||
// or nil if the file is empty
|
||||
func (s *fsStore) sourceFromFile(relPath string) (checkpoint.RemoteConfigSource, error) {
|
||||
path := filepath.Join(s.checkpointsDir, relPath)
|
||||
data, err := s.fs.ReadFile(path)
|
||||
func (s *fsStore) checkpointPath(uid, resourceVersion string) string {
|
||||
return filepath.Join(s.dir, checkpointsDir, uid, resourceVersion)
|
||||
}
|
||||
|
||||
func (s *fsStore) metaPath(name string) string {
|
||||
return filepath.Join(s.dir, metaDir, name)
|
||||
}
|
||||
|
||||
func readRemoteConfigSource(fs utilfs.Filesystem, path string) (checkpoint.RemoteConfigSource, error) {
|
||||
data, err := fs.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if len(data) == 0 {
|
||||
@ -141,17 +172,23 @@ func (s *fsStore) sourceFromFile(relPath string) (checkpoint.RemoteConfigSource,
|
||||
return checkpoint.DecodeRemoteConfigSource(data)
|
||||
}
|
||||
|
||||
// set source file replaces the file at `s.checkpointsDir/relPath` with a file containing `source`
|
||||
func (s *fsStore) setSourceFile(relPath string, source checkpoint.RemoteConfigSource) error {
|
||||
path := filepath.Join(s.checkpointsDir, relPath)
|
||||
func writeRemoteConfigSource(fs utilfs.Filesystem, path string, source checkpoint.RemoteConfigSource) error {
|
||||
// if nil, reset the file
|
||||
if source == nil {
|
||||
return utilfiles.ReplaceFile(s.fs, path, []byte{})
|
||||
return utilfiles.ReplaceFile(fs, path, []byte{})
|
||||
}
|
||||
// check that UID and ResourceVersion are non-empty,
|
||||
// error to save reference if the checkpoint can't be fully resolved
|
||||
if source.UID() == "" {
|
||||
return fmt.Errorf("failed to write RemoteConfigSource, empty UID is ambiguous")
|
||||
}
|
||||
if source.ResourceVersion() == "" {
|
||||
return fmt.Errorf("failed to write RemoteConfigSource, empty ResourceVersion is ambiguous")
|
||||
}
|
||||
// encode the source and save it to the file
|
||||
data, err := source.Encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return utilfiles.ReplaceFile(s.fs, path, data)
|
||||
return utilfiles.ReplaceFile(fs, path, data)
|
||||
}
|
||||
|
857
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go
generated
vendored
857
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/fsstore_test.go
generated
vendored
File diff suppressed because it is too large
Load Diff
70
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/store.go
generated
vendored
70
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/store.go
generated
vendored
@ -20,65 +20,51 @@ import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint"
|
||||
)
|
||||
|
||||
// Store saves checkpoints and information about which is the current and last-known-good checkpoint to a storage layer
|
||||
// Store saves checkpoints and information about which is the assigned and last-known-good checkpoint to a storage layer
|
||||
type Store interface {
|
||||
// Initialize sets up the storage layer
|
||||
Initialize() error
|
||||
// Exists returns true if a checkpoint with `uid` exists in the store, false otherwise
|
||||
Exists(uid string) (bool, error)
|
||||
// Save saves the checkpoint to the storage layer
|
||||
Save(c checkpoint.Checkpoint) error
|
||||
// Load loads the checkpoint with UID `uid` from the storage layer, or returns an error if the checkpoint does not exist
|
||||
Load(uid string) (checkpoint.Checkpoint, error)
|
||||
// CurrentModified returns the last time that the current UID was set
|
||||
CurrentModified() (time.Time, error)
|
||||
// Current returns the source that points to the current checkpoint, or nil if no current checkpoint is set
|
||||
Current() (checkpoint.RemoteConfigSource, error)
|
||||
|
||||
// Exists returns true if the object referenced by `source` has been checkpointed.
|
||||
// The source must be unambiguous - e.g. if referencing an API object it must specify both uid and resourceVersion.
|
||||
Exists(source checkpoint.RemoteConfigSource) (bool, error)
|
||||
// Save Kubelet config payloads to the storage layer. It must be possible to unmarshal the payload to a KubeletConfiguration.
|
||||
// The following payload types are supported:
|
||||
// - k8s.io/api/core/v1.ConfigMap
|
||||
Save(c checkpoint.Payload) error
|
||||
// Load loads the KubeletConfiguration from the checkpoint referenced by `source`.
|
||||
Load(source checkpoint.RemoteConfigSource) (*kubeletconfig.KubeletConfiguration, error)
|
||||
|
||||
// AssignedModified returns the last time that the assigned checkpoint was set
|
||||
AssignedModified() (time.Time, error)
|
||||
// Assigned returns the source that points to the checkpoint currently assigned to the Kubelet, or nil if no assigned checkpoint is set
|
||||
Assigned() (checkpoint.RemoteConfigSource, error)
|
||||
// LastKnownGood returns the source that points to the last-known-good checkpoint, or nil if no last-known-good checkpoint is set
|
||||
LastKnownGood() (checkpoint.RemoteConfigSource, error)
|
||||
// SetCurrent saves the source that points to the current checkpoint, set to nil to unset
|
||||
SetCurrent(source checkpoint.RemoteConfigSource) error
|
||||
// SetCurrentUpdated is similar to SetCurrent, but also returns whether the current checkpoint changed as a result
|
||||
SetCurrentUpdated(source checkpoint.RemoteConfigSource) (bool, error)
|
||||
|
||||
// SetAssigned saves the source that points to the assigned checkpoint, set to nil to unset
|
||||
SetAssigned(source checkpoint.RemoteConfigSource) error
|
||||
// SetLastKnownGood saves the source that points to the last-known-good checkpoint, set to nil to unset
|
||||
SetLastKnownGood(source checkpoint.RemoteConfigSource) error
|
||||
// Reset unsets the current and last-known-good UIDs and returns whether the current UID was unset as a result of the reset
|
||||
// Reset unsets the assigned and last-known-good checkpoints and returns whether the assigned checkpoint was unset as a result of the reset
|
||||
Reset() (bool, error)
|
||||
}
|
||||
|
||||
// reset is a helper for implementing Reset, which can be implemented in terms of Store methods
|
||||
func reset(s Store) (bool, error) {
|
||||
assigned, err := s.Assigned()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.SetLastKnownGood(nil); err != nil {
|
||||
return false, fmt.Errorf("failed to reset last-known-good UID in checkpoint store, error: %v", err)
|
||||
}
|
||||
updated, err := s.SetCurrentUpdated(nil)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to reset current UID in checkpoint store, error: %v", err)
|
||||
if err := s.SetAssigned(nil); err != nil {
|
||||
return false, fmt.Errorf("failed to reset assigned UID in checkpoint store, error: %v", err)
|
||||
}
|
||||
return updated, nil
|
||||
}
|
||||
|
||||
// setCurrentUpdated is a helper for implementing SetCurrentUpdated, which can be implemented in terms of Store methods
|
||||
func setCurrentUpdated(s Store, source checkpoint.RemoteConfigSource) (bool, error) {
|
||||
cur, err := s.Current()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// if both are nil, no need to update
|
||||
if cur == nil && source == nil {
|
||||
return false, nil
|
||||
}
|
||||
// if UIDs match, no need to update
|
||||
if (source != nil && cur != nil) && cur.UID() == source.UID() {
|
||||
return false, nil
|
||||
}
|
||||
// update the source
|
||||
if err := s.SetCurrent(source); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
return assigned != nil, nil
|
||||
}
|
||||
|
67
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go
generated
vendored
67
vendor/k8s.io/kubernetes/pkg/kubelet/kubeletconfig/checkpoint/store/store_test.go
generated
vendored
@ -26,11 +26,21 @@ import (
|
||||
)
|
||||
|
||||
func TestReset(t *testing.T) {
|
||||
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
|
||||
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "name",
|
||||
Namespace: "namespace",
|
||||
UID: "uid",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
|
||||
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMap: &apiv1.ConfigMapNodeConfigSource{
|
||||
Name: "other-name",
|
||||
Namespace: "namespace",
|
||||
UID: "other-uid",
|
||||
KubeletConfigKey: "kubelet",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
@ -38,59 +48,24 @@ func TestReset(t *testing.T) {
|
||||
s *fakeStore
|
||||
updated bool
|
||||
}{
|
||||
{&fakeStore{current: nil, lastKnownGood: nil}, false},
|
||||
{&fakeStore{current: source, lastKnownGood: nil}, true},
|
||||
{&fakeStore{current: nil, lastKnownGood: source}, false},
|
||||
{&fakeStore{current: source, lastKnownGood: source}, true},
|
||||
{&fakeStore{current: source, lastKnownGood: otherSource}, true},
|
||||
{&fakeStore{current: otherSource, lastKnownGood: source}, true},
|
||||
{&fakeStore{assigned: nil, lastKnownGood: nil}, false},
|
||||
{&fakeStore{assigned: source, lastKnownGood: nil}, true},
|
||||
{&fakeStore{assigned: nil, lastKnownGood: source}, false},
|
||||
{&fakeStore{assigned: source, lastKnownGood: source}, true},
|
||||
{&fakeStore{assigned: source, lastKnownGood: otherSource}, true},
|
||||
{&fakeStore{assigned: otherSource, lastKnownGood: source}, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
updated, err := reset(c.s)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if c.s.current != nil || c.s.lastKnownGood != nil {
|
||||
t.Errorf("case %q, expect nil for current and last-known-good checkpoints, but still have %q and %q, respectively",
|
||||
spew.Sdump(c.s), c.s.current, c.s.lastKnownGood)
|
||||
if c.s.assigned != nil || c.s.lastKnownGood != nil {
|
||||
t.Errorf("case %q, expect nil for assigned and last-known-good checkpoints, but still have %q and %q, respectively",
|
||||
spew.Sdump(c.s), c.s.assigned, c.s.lastKnownGood)
|
||||
}
|
||||
if c.updated != updated {
|
||||
t.Errorf("case %q, expect reset to return %t, but got %t", spew.Sdump(c.s), c.updated, updated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetCurrentUpdated(t *testing.T) {
|
||||
source, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "name", Namespace: "namespace", UID: "uid"}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
otherSource, _, err := checkpoint.NewRemoteConfigSource(&apiv1.NodeConfigSource{ConfigMapRef: &apiv1.ObjectReference{Name: "other-name", Namespace: "namespace", UID: "other-uid"}})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
cases := []struct {
|
||||
s *fakeStore
|
||||
newCurrent checkpoint.RemoteConfigSource
|
||||
updated bool
|
||||
}{
|
||||
{&fakeStore{current: nil}, nil, false},
|
||||
{&fakeStore{current: nil}, source, true},
|
||||
{&fakeStore{current: source}, source, false},
|
||||
{&fakeStore{current: source}, nil, true},
|
||||
{&fakeStore{current: source}, otherSource, true},
|
||||
}
|
||||
for _, c := range cases {
|
||||
current := c.s.current
|
||||
updated, err := setCurrentUpdated(c.s, c.newCurrent)
|
||||
if err != nil {
|
||||
t.Fatalf("case %q -> %q, unexpected error: %v", current, c.newCurrent, err)
|
||||
}
|
||||
if c.newCurrent != c.s.current {
|
||||
t.Errorf("case %q -> %q, expect current UID to be %q, but got %q", current, c.newCurrent, c.newCurrent, c.s.current)
|
||||
}
|
||||
if c.updated != updated {
|
||||
t.Errorf("case %q -> %q, expect setCurrentUpdated to return %t, but got %t", current, c.newCurrent, c.updated, updated)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
Reference in New Issue
Block a user