build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>
This commit is contained in:
Niels de Vos
2025-03-04 08:57:28 +01:00
committed by mergify[bot]
parent 15da101b1b
commit bec6090996
8047 changed files with 1407827 additions and 3453 deletions

View File

@ -0,0 +1,389 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbor
import (
"bytes"
"encoding/hex"
"errors"
"fmt"
"io"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes"
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
util "k8s.io/apimachinery/pkg/util/runtime"
"github.com/fxamacker/cbor/v2"
)
type metaFactory interface {
// Interpret should return the version and kind of the wire-format of the object.
Interpret(data []byte) (*schema.GroupVersionKind, error)
}
type defaultMetaFactory struct{}
func (mf *defaultMetaFactory) Interpret(data []byte) (*schema.GroupVersionKind, error) {
var tm metav1.TypeMeta
// The input is expected to include additional map keys besides apiVersion and kind, so use
// lax mode for decoding into TypeMeta.
if err := modes.DecodeLax.Unmarshal(data, &tm); err != nil {
return nil, fmt.Errorf("unable to determine group/version/kind: %w", err)
}
actual := tm.GetObjectKind().GroupVersionKind()
return &actual, nil
}
type Serializer interface {
runtime.Serializer
runtime.NondeterministicEncoder
recognizer.RecognizingDecoder
// NewSerializer returns a value of this interface type rather than exporting the serializer
// type and returning one of those because the zero value of serializer isn't ready to
// use. Users aren't intended to implement cbor.Serializer themselves, and this unexported
// interface method is here to prevent that (https://go.dev/blog/module-compatibility).
private()
}
var _ Serializer = &serializer{}
type options struct {
strict bool
transcode bool
}
type Option func(*options)
// Strict configures a serializer to return a strict decoding error when it encounters map keys that
// do not correspond to a field in the target object of a decode operation. This option is disabled
// by default.
func Strict(s bool) Option {
return func(opts *options) {
opts.strict = s
}
}
// Transcode configures a serializer to transcode the "raw" bytes of a decoded runtime.RawExtension
// or metav1.FieldsV1 object to JSON. This is enabled by default to support existing programs that
// depend on the assumption that objects of either type contain valid JSON.
func Transcode(s bool) Option {
return func(opts *options) {
opts.transcode = s
}
}
type serializer struct {
metaFactory metaFactory
creater runtime.ObjectCreater
typer runtime.ObjectTyper
options options
}
func (serializer) private() {}
// NewSerializer creates and returns a serializer configured with the provided options. The default
// options are equivalent to explicitly passing Strict(false) and Transcode(true).
func NewSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper, options ...Option) Serializer {
return newSerializer(&defaultMetaFactory{}, creater, typer, options...)
}
func newSerializer(metaFactory metaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, options ...Option) *serializer {
s := &serializer{
metaFactory: metaFactory,
creater: creater,
typer: typer,
}
s.options.transcode = true
for _, o := range options {
o(&s.options)
}
return s
}
func (s *serializer) Identifier() runtime.Identifier {
return "cbor"
}
// Encode writes a CBOR representation of the given object.
//
// Because the CBOR data item written by a call to Encode is always enclosed in the "self-described
// CBOR" tag, its encoded form always has the prefix 0xd9d9f7. This prefix is suitable for use as a
// "magic number" for distinguishing encoded CBOR from other protocols.
//
// The default serialization behavior for any given object replicates the behavior of the JSON
// serializer as far as it is necessary to allow the CBOR serializer to be used as a drop-in
// replacement for the JSON serializer, with limited exceptions. For example, the distinction
// between integers and floating-point numbers is preserved in CBOR due to its distinct
// representations for each type.
//
// Objects implementing runtime.Unstructured will have their unstructured content encoded rather
// than following the default behavior for their dynamic type.
func (s *serializer) Encode(obj runtime.Object, w io.Writer) error {
return s.encode(modes.Encode, obj, w)
}
func (s *serializer) EncodeNondeterministic(obj runtime.Object, w io.Writer) error {
return s.encode(modes.EncodeNondeterministic, obj, w)
}
func (s *serializer) encode(mode modes.EncMode, obj runtime.Object, w io.Writer) error {
var v interface{} = obj
if u, ok := obj.(runtime.Unstructured); ok {
v = u.UnstructuredContent()
}
if err := modes.RejectCustomMarshalers(v); err != nil {
return err
}
if _, err := w.Write(selfDescribedCBOR); err != nil {
return err
}
return mode.MarshalTo(v, w)
}
// gvkWithDefaults returns group kind and version defaulting from provided default
func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind {
if len(actual.Kind) == 0 {
actual.Kind = defaultGVK.Kind
}
if len(actual.Version) == 0 && len(actual.Group) == 0 {
actual.Group = defaultGVK.Group
actual.Version = defaultGVK.Version
}
if len(actual.Version) == 0 && actual.Group == defaultGVK.Group {
actual.Version = defaultGVK.Version
}
return actual
}
// diagnose returns the diagnostic encoding of a well-formed CBOR data item.
func diagnose(data []byte) string {
diag, err := modes.Diagnostic.Diagnose(data)
if err != nil {
// Since the input must already be well-formed CBOR, converting it to diagnostic
// notation should not fail.
util.HandleError(err)
return hex.EncodeToString(data)
}
return diag
}
// unmarshal unmarshals CBOR data from the provided byte slice into a Go object. If the decoder is
// configured to report strict errors, the first error return value may be a non-nil strict decoding
// error. If the last error return value is non-nil, then the unmarshal failed entirely and the
// state of the destination object should not be relied on.
func (s *serializer) unmarshal(data []byte, into interface{}) (strict, lax error) {
if u, ok := into.(runtime.Unstructured); ok {
var content map[string]interface{}
defer func() {
switch u := u.(type) {
case *unstructured.UnstructuredList:
// UnstructuredList's implementation of SetUnstructuredContent
// produces different objects than those produced by a decode using
// UnstructuredJSONScheme:
//
// 1. SetUnstructuredContent retains the "items" key in the list's
// Object field. It is omitted from Object when decoding with
// UnstructuredJSONScheme.
// 2. SetUnstructuredContent does not populate "apiVersion" and
// "kind" on each entry of its Items
// field. UnstructuredJSONScheme does, inferring the singular
// Kind from the list Kind.
// 3. SetUnstructuredContent ignores entries of "items" that are
// not JSON objects or are objects without
// "kind". UnstructuredJSONScheme returns an error in either
// case.
//
// UnstructuredJSONScheme's behavior is replicated here.
var items []interface{}
if uncast, present := content["items"]; present {
var cast bool
items, cast = uncast.([]interface{})
if !cast {
strict, lax = nil, fmt.Errorf("items field of UnstructuredList must be encoded as an array or null if present")
return
}
}
apiVersion, _ := content["apiVersion"].(string)
kind, _ := content["kind"].(string)
kind = strings.TrimSuffix(kind, "List")
var unstructureds []unstructured.Unstructured
if len(items) > 0 {
unstructureds = make([]unstructured.Unstructured, len(items))
}
for i := range items {
object, cast := items[i].(map[string]interface{})
if !cast {
strict, lax = nil, fmt.Errorf("elements of the items field of UnstructuredList must be encoded as a map")
return
}
// As in UnstructuredJSONScheme, only set the heuristic
// singular GVK when both "apiVersion" and "kind" are either
// missing, non-string, or empty.
object["apiVersion"], _ = object["apiVersion"].(string)
object["kind"], _ = object["kind"].(string)
if object["apiVersion"] == "" && object["kind"] == "" {
object["apiVersion"] = apiVersion
object["kind"] = kind
}
if object["kind"] == "" {
strict, lax = nil, runtime.NewMissingKindErr(diagnose(data))
return
}
if object["apiVersion"] == "" {
strict, lax = nil, runtime.NewMissingVersionErr(diagnose(data))
return
}
unstructureds[i].Object = object
}
delete(content, "items")
u.Object = content
u.Items = unstructureds
default:
u.SetUnstructuredContent(content)
}
}()
into = &content
} else if err := modes.RejectCustomMarshalers(into); err != nil {
return nil, err
}
if !s.options.strict {
return nil, modes.DecodeLax.Unmarshal(data, into)
}
err := modes.Decode.Unmarshal(data, into)
// TODO: UnknownFieldError is ambiguous. It only provides the index of the first problematic
// map entry encountered and does not indicate which map the index refers to.
var unknownField *cbor.UnknownFieldError
if errors.As(err, &unknownField) {
// Unlike JSON, there are no strict errors in CBOR for duplicate map keys. CBOR maps
// with duplicate keys are considered invalid according to the spec and are rejected
// entirely.
return runtime.NewStrictDecodingError([]error{unknownField}), modes.DecodeLax.Unmarshal(data, into)
}
return nil, err
}
func (s *serializer) Decode(data []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
// A preliminary pass over the input to obtain the actual GVK is redundant on a successful
// decode into Unstructured.
if _, ok := into.(runtime.Unstructured); ok {
if _, unmarshalErr := s.unmarshal(data, into); unmarshalErr != nil {
actual, interpretErr := s.metaFactory.Interpret(data)
if interpretErr != nil {
return nil, nil, interpretErr
}
if gvk != nil {
*actual = gvkWithDefaults(*actual, *gvk)
}
return nil, actual, unmarshalErr
}
actual := into.GetObjectKind().GroupVersionKind()
if len(actual.Kind) == 0 {
return nil, &actual, runtime.NewMissingKindErr(diagnose(data))
}
if len(actual.Version) == 0 {
return nil, &actual, runtime.NewMissingVersionErr(diagnose(data))
}
return into, &actual, nil
}
actual, err := s.metaFactory.Interpret(data)
if err != nil {
return nil, nil, err
}
if gvk != nil {
*actual = gvkWithDefaults(*actual, *gvk)
}
if into != nil {
types, _, err := s.typer.ObjectKinds(into)
if err != nil {
return nil, actual, err
}
*actual = gvkWithDefaults(*actual, types[0])
}
if len(actual.Kind) == 0 {
return nil, actual, runtime.NewMissingKindErr(diagnose(data))
}
if len(actual.Version) == 0 {
return nil, actual, runtime.NewMissingVersionErr(diagnose(data))
}
obj, err := runtime.UseOrCreateObject(s.typer, s.creater, *actual, into)
if err != nil {
return nil, actual, err
}
strict, err := s.unmarshal(data, obj)
if err != nil {
return nil, actual, err
}
if s.options.transcode {
if err := transcodeRawTypes(obj); err != nil {
return nil, actual, err
}
}
return obj, actual, strict
}
// selfDescribedCBOR is the CBOR encoding of the head of tag number 55799. This tag, specified in
// RFC 8949 Section 3.4.6 "Self-Described CBOR", encloses all output from the encoder, has no
// special semantics, and is used as a magic number to recognize CBOR-encoded data items.
//
// See https://www.rfc-editor.org/rfc/rfc8949.html#name-self-described-cbor.
var selfDescribedCBOR = []byte{0xd9, 0xd9, 0xf7}
func (s *serializer) RecognizesData(data []byte) (ok, unknown bool, err error) {
return bytes.HasPrefix(data, selfDescribedCBOR), false, nil
}
// NewSerializerInfo returns a default SerializerInfo for CBOR using the given creater and typer.
func NewSerializerInfo(creater runtime.ObjectCreater, typer runtime.ObjectTyper) runtime.SerializerInfo {
return runtime.SerializerInfo{
MediaType: "application/cbor",
MediaTypeType: "application",
MediaTypeSubType: "cbor",
Serializer: NewSerializer(creater, typer),
StrictSerializer: NewSerializer(creater, typer, Strict(true)),
StreamSerializer: &runtime.StreamSerializerInfo{
Framer: NewFramer(),
Serializer: NewSerializer(creater, typer, Transcode(false)),
},
}
}

View File

@ -0,0 +1,61 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package direct provides functions for marshaling and unmarshaling between arbitrary Go values and
// CBOR data, with behavior that is compatible with that of the CBOR serializer. In particular,
// types that implement cbor.Marshaler and cbor.Unmarshaler should use these functions.
package direct
import (
"k8s.io/apimachinery/pkg/runtime/serializer/cbor/internal/modes"
)
// Marshal serializes a value to CBOR. If there is more than one way to encode the value, it will
// make the same choice as the CBOR implementation of runtime.Serializer.
//
// Note: Support for CBOR is at an alpha stage. If the value (or, for composite types, any of its
// nested values) implement any of the interfaces encoding.TextMarshaler, encoding.TextUnmarshaler,
// encoding/json.Marshaler, or encoding/json.Unmarshaler, a non-nil error will be returned unless
// the value also implements the corresponding CBOR interfaces. This limitation will ultimately be
// removed in favor of automatic transcoding to CBOR.
func Marshal(src interface{}) ([]byte, error) {
if err := modes.RejectCustomMarshalers(src); err != nil {
return nil, err
}
return modes.Encode.Marshal(src)
}
// Unmarshal deserializes from CBOR into an addressable value. If there is more than one way to
// unmarshal a value, it will make the same choice as the CBOR implementation of runtime.Serializer.
//
// Note: Support for CBOR is at an alpha stage. If the value (or, for composite types, any of its
// nested values) implement any of the interfaces encoding.TextMarshaler, encoding.TextUnmarshaler,
// encoding/json.Marshaler, or encoding/json.Unmarshaler, a non-nil error will be returned unless
// the value also implements the corresponding CBOR interfaces. This limitation will ultimately be
// removed in favor of automatic transcoding to CBOR.
func Unmarshal(src []byte, dst interface{}) error {
if err := modes.RejectCustomMarshalers(dst); err != nil {
return err
}
return modes.Decode.Unmarshal(src, dst)
}
// Diagnose accepts well-formed CBOR bytes and returns a string representing the same data item in
// human-readable diagnostic notation (RFC 8949 Section 8). The diagnostic notation is not meant to
// be parsed.
func Diagnose(src []byte) (string, error) {
return modes.Diagnostic.Diagnose(src)
}

View File

@ -0,0 +1,90 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbor
import (
"io"
"k8s.io/apimachinery/pkg/runtime"
"github.com/fxamacker/cbor/v2"
)
// NewFramer returns a runtime.Framer based on RFC 8742 CBOR Sequences. Each frame contains exactly
// one encoded CBOR data item.
func NewFramer() runtime.Framer {
return framer{}
}
var _ runtime.Framer = framer{}
type framer struct{}
func (framer) NewFrameReader(rc io.ReadCloser) io.ReadCloser {
return &frameReader{
decoder: cbor.NewDecoder(rc),
closer: rc,
}
}
func (framer) NewFrameWriter(w io.Writer) io.Writer {
// Each data item in a CBOR sequence is self-delimiting (like JSON objects).
return w
}
type frameReader struct {
decoder *cbor.Decoder
closer io.Closer
overflow []byte
}
func (fr *frameReader) Read(dst []byte) (int, error) {
if len(fr.overflow) > 0 {
// We read a frame that was too large for the destination slice in a previous call
// to Read and have bytes left over.
n := copy(dst, fr.overflow)
if n < len(fr.overflow) {
fr.overflow = fr.overflow[n:]
return n, io.ErrShortBuffer
}
fr.overflow = nil
return n, nil
}
// The Reader contract allows implementations to use all of dst[0:len(dst)] as scratch
// space, even if n < len(dst), but it does not allow implementations to use
// dst[len(dst):cap(dst)]. Slicing it up-front allows us to append to it without worrying
// about overwriting dst[len(dst):cap(dst)].
m := cbor.RawMessage(dst[0:0:len(dst)])
if err := fr.decoder.Decode(&m); err != nil {
return 0, err
}
if len(m) > len(dst) {
// The frame was too big, m has a newly-allocated underlying array to accommodate
// it.
fr.overflow = m[len(dst):]
return copy(dst, m), io.ErrShortBuffer
}
return len(m), nil
}
func (fr *frameReader) Close() error {
return fr.closer.Close()
}

View File

@ -0,0 +1,65 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package modes
import (
"bytes"
"sync"
)
var buffers = BufferProvider{p: new(sync.Pool)}
type buffer struct {
bytes.Buffer
}
type pool interface {
Get() interface{}
Put(interface{})
}
type BufferProvider struct {
p pool
}
func (b *BufferProvider) Get() *buffer {
if buf, ok := b.p.Get().(*buffer); ok {
return buf
}
return &buffer{}
}
func (b *BufferProvider) Put(buf *buffer) {
if buf.Cap() > 3*1024*1024 /* Default MaxRequestBodyBytes */ {
// Objects in a sync.Pool are assumed to be fungible. This is not a good assumption
// for pools of *bytes.Buffer because a *bytes.Buffer's underlying array grows as
// needed to accommodate writes. In Kubernetes, apiservers tend to encode "small"
// objects very frequently and much larger objects (especially large lists) only
// occasionally. Under steady load, pooled buffers tend to be borrowed frequently
// enough to prevent them from being released. Over time, each buffer is used to
// encode a large object and its capacity increases accordingly. The result is that
// practically all buffers in the pool retain much more capacity than needed to
// encode most objects.
// As a basic mitigation for the worst case, buffers with more capacity than the
// default max request body size are never returned to the pool.
// TODO: Optimize for higher buffer utilization.
return
}
buf.Reset()
b.p.Put(buf)
}

View File

@ -0,0 +1,422 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package modes
import (
"encoding"
"encoding/json"
"errors"
"fmt"
"reflect"
"sync"
"github.com/fxamacker/cbor/v2"
)
// Returns a non-nil error if and only if the argument's type (or one of its component types, for
// composite types) implements json.Marshaler or encoding.TextMarshaler without also implementing
// cbor.Marshaler and likewise for the respective Unmarshaler interfaces.
//
// This is a temporary, graduation-blocking restriction and will be removed in favor of automatic
// transcoding between CBOR and JSON/text for these types. This restriction allows CBOR to be
// exercised for in-tree and unstructured types while mitigating the risk of mangling out-of-tree
// types in client programs.
func RejectCustomMarshalers(v interface{}) error {
if v == nil {
return nil
}
rv := reflect.ValueOf(v)
if err := marshalerCache.getChecker(rv.Type()).check(rv, maxDepth); err != nil {
return fmt.Errorf("unable to serialize %T: %w", v, err)
}
if err := unmarshalerCache.getChecker(rv.Type()).check(rv, maxDepth); err != nil {
return fmt.Errorf("unable to serialize %T: %w", v, err)
}
return nil
}
// Recursion depth is limited as a basic mitigation against cyclic objects. Objects created by the
// decoder shouldn't be able to contain cycles, but practically any object can be passed to the
// encoder.
var errMaxDepthExceeded = errors.New("object depth exceeds limit (possible cycle?)")
// The JSON encoder begins detecting cycles after depth 1000. Use a generous limit here, knowing
// that it can might deeply nested acyclic objects. The limit will be removed along with the rest of
// this mechanism.
const maxDepth = 2048
var marshalerCache = checkers{
cborInterface: reflect.TypeFor[cbor.Marshaler](),
nonCBORInterfaces: []reflect.Type{
reflect.TypeFor[json.Marshaler](),
reflect.TypeFor[encoding.TextMarshaler](),
},
}
var unmarshalerCache = checkers{
cborInterface: reflect.TypeFor[cbor.Unmarshaler](),
nonCBORInterfaces: []reflect.Type{
reflect.TypeFor[json.Unmarshaler](),
reflect.TypeFor[encoding.TextUnmarshaler](),
},
assumeAddressableValues: true,
}
// checker wraps a function for dynamically checking a value of a specific type for custom JSON
// behaviors not matched by a custom CBOR behavior.
type checker struct {
// check returns a non-nil error if the given value might be marshalled to or from CBOR
// using the default behavior for its kind, but marshalled to or from JSON using custom
// behavior.
check func(rv reflect.Value, depth int) error
// safe returns true if all values of this type are safe from mismatched custom marshalers.
safe func() bool
}
// TODO: stale
// Having a single addressable checker for comparisons lets us prune and collapse parts of the
// object traversal that are statically known to be safe. Depending on the type, it may be
// unnecessary to inspect each value of that type. For example, no value of the built-in type bool
// can implement json.Marshaler (a named type whose underlying type is bool could, but it is a
// distinct type from bool).
var noop = checker{
safe: func() bool {
return true
},
check: func(rv reflect.Value, depth int) error {
return nil
},
}
type checkers struct {
m sync.Map // reflect.Type => *checker
cborInterface reflect.Type
nonCBORInterfaces []reflect.Type
assumeAddressableValues bool
}
func (cache *checkers) getChecker(rt reflect.Type) checker {
if ptr, ok := cache.m.Load(rt); ok {
return *ptr.(*checker)
}
return cache.getCheckerInternal(rt, nil)
}
// linked list node representing the path from a composite type to an element type
type path struct {
Type reflect.Type
Parent *path
}
func (p path) cyclic(rt reflect.Type) bool {
for ancestor := &p; ancestor != nil; ancestor = ancestor.Parent {
if ancestor.Type == rt {
return true
}
}
return false
}
func (cache *checkers) getCheckerInternal(rt reflect.Type, parent *path) (c checker) {
// Store a placeholder cache entry first to handle cyclic types.
var wg sync.WaitGroup
wg.Add(1)
defer wg.Done()
c = checker{
safe: func() bool {
wg.Wait()
return c.safe()
},
check: func(rv reflect.Value, depth int) error {
wg.Wait()
return c.check(rv, depth)
},
}
if actual, loaded := cache.m.LoadOrStore(rt, &c); loaded {
// Someone else stored an entry for this type, use it.
return *actual.(*checker)
}
// Take a nonreflective path for the unstructured container types. They're common and
// usually nested inside one another.
switch rt {
case reflect.TypeFor[map[string]interface{}](), reflect.TypeFor[[]interface{}]():
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
return checkUnstructuredValue(cache, rv.Interface(), depth)
},
}
}
// It's possible that one of the relevant interfaces is implemented on a type with a pointer
// receiver, but that a particular value of that type is not addressable. For example:
//
// func (Foo) MarshalText() ([]byte, error) { ... }
// func (*Foo) MarshalCBOR() ([]byte, error) { ... }
//
// Both methods are in the method set of *Foo, but the method set of Foo contains only
// MarshalText.
//
// Both the unmarshaler and marshaler checks assume that methods implementing a JSON or text
// interface with a pointer receiver are always accessible. Only the unmarshaler check
// assumes that CBOR methods with pointer receivers are accessible.
if rt.Implements(cache.cborInterface) {
return noop
}
for _, unsafe := range cache.nonCBORInterfaces {
if rt.Implements(unsafe) {
err := fmt.Errorf("%v implements %v without corresponding cbor interface", rt, unsafe)
return checker{
safe: func() bool {
return false
},
check: func(reflect.Value, int) error {
return err
},
}
}
}
if cache.assumeAddressableValues && reflect.PointerTo(rt).Implements(cache.cborInterface) {
return noop
}
for _, unsafe := range cache.nonCBORInterfaces {
if reflect.PointerTo(rt).Implements(unsafe) {
err := fmt.Errorf("%v implements %v without corresponding cbor interface", reflect.PointerTo(rt), unsafe)
return checker{
safe: func() bool {
return false
},
check: func(reflect.Value, int) error {
return err
},
}
}
}
self := &path{Type: rt, Parent: parent}
switch rt.Kind() {
case reflect.Array:
ce := cache.getCheckerInternal(rt.Elem(), self)
rtlen := rt.Len()
if rtlen == 0 || (!self.cyclic(rt.Elem()) && ce.safe()) {
return noop
}
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if depth <= 0 {
return errMaxDepthExceeded
}
for i := 0; i < rtlen; i++ {
if err := ce.check(rv.Index(i), depth-1); err != nil {
return err
}
}
return nil
},
}
case reflect.Interface:
// All interface values have to be checked because their dynamic type might
// implement one of the interesting interfaces or be composed of another type that
// does.
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if rv.IsNil() {
return nil
}
// Unpacking interfaces must count against recursion depth,
// consider this cycle:
// > var i interface{}
// > var p *interface{} = &i
// > i = p
// > rv := reflect.ValueOf(i)
// > for {
// > rv = rv.Elem()
// > }
if depth <= 0 {
return errMaxDepthExceeded
}
rv = rv.Elem()
return cache.getChecker(rv.Type()).check(rv, depth-1)
},
}
case reflect.Map:
rtk := rt.Key()
ck := cache.getCheckerInternal(rtk, self)
rte := rt.Elem()
ce := cache.getCheckerInternal(rte, self)
if !self.cyclic(rtk) && !self.cyclic(rte) && ck.safe() && ce.safe() {
return noop
}
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if depth <= 0 {
return errMaxDepthExceeded
}
iter := rv.MapRange()
rvk := reflect.New(rtk).Elem()
rve := reflect.New(rte).Elem()
for iter.Next() {
rvk.SetIterKey(iter)
if err := ck.check(rvk, depth-1); err != nil {
return err
}
rve.SetIterValue(iter)
if err := ce.check(rve, depth-1); err != nil {
return err
}
}
return nil
},
}
case reflect.Pointer:
ce := cache.getCheckerInternal(rt.Elem(), self)
if !self.cyclic(rt.Elem()) && ce.safe() {
return noop
}
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if rv.IsNil() {
return nil
}
if depth <= 0 {
return errMaxDepthExceeded
}
return ce.check(rv.Elem(), depth-1)
},
}
case reflect.Slice:
ce := cache.getCheckerInternal(rt.Elem(), self)
if !self.cyclic(rt.Elem()) && ce.safe() {
return noop
}
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if depth <= 0 {
return errMaxDepthExceeded
}
for i := 0; i < rv.Len(); i++ {
if err := ce.check(rv.Index(i), depth-1); err != nil {
return err
}
}
return nil
},
}
case reflect.Struct:
type field struct {
Index int
Checker checker
}
var fields []field
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
cf := cache.getCheckerInternal(f.Type, self)
if !self.cyclic(f.Type) && cf.safe() {
continue
}
fields = append(fields, field{Index: i, Checker: cf})
}
if len(fields) == 0 {
return noop
}
return checker{
safe: func() bool {
return false
},
check: func(rv reflect.Value, depth int) error {
if depth <= 0 {
return errMaxDepthExceeded
}
for _, fi := range fields {
if err := fi.Checker.check(rv.Field(fi.Index), depth-1); err != nil {
return err
}
}
return nil
},
}
default:
// Not a serializable composite type (funcs and channels are composite types but are
// rejected by JSON and CBOR serialization).
return noop
}
}
func checkUnstructuredValue(cache *checkers, v interface{}, depth int) error {
switch v := v.(type) {
case nil, bool, int64, float64, string:
return nil
case []interface{}:
if depth <= 0 {
return errMaxDepthExceeded
}
for _, element := range v {
if err := checkUnstructuredValue(cache, element, depth-1); err != nil {
return err
}
}
return nil
case map[string]interface{}:
if depth <= 0 {
return errMaxDepthExceeded
}
for _, element := range v {
if err := checkUnstructuredValue(cache, element, depth-1); err != nil {
return err
}
}
return nil
default:
// Unmarshaling an unstructured doesn't use other dynamic types, but nothing
// prevents inserting values with arbitrary dynamic types into unstructured content,
// as long as they can be marshalled.
rv := reflect.ValueOf(v)
return cache.getChecker(rv.Type()).check(rv, depth)
}
}

View File

@ -0,0 +1,158 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package modes
import (
"reflect"
"github.com/fxamacker/cbor/v2"
)
var simpleValues *cbor.SimpleValueRegistry = func() *cbor.SimpleValueRegistry {
var opts []func(*cbor.SimpleValueRegistry) error
for sv := 0; sv <= 255; sv++ {
// Reject simple values 0-19, 23, and 32-255. The simple values 24-31 are reserved
// and considered ill-formed by the CBOR specification. We only accept false (20),
// true (21), and null (22).
switch sv {
case 20: // false
case 21: // true
case 22: // null
case 24, 25, 26, 27, 28, 29, 30, 31: // reserved
default:
opts = append(opts, cbor.WithRejectedSimpleValue(cbor.SimpleValue(sv)))
}
}
simpleValues, err := cbor.NewSimpleValueRegistryFromDefaults(opts...)
if err != nil {
panic(err)
}
return simpleValues
}()
var Decode cbor.DecMode = func() cbor.DecMode {
decode, err := cbor.DecOptions{
// Maps with duplicate keys are well-formed but invalid according to the CBOR spec
// and never acceptable. Unlike the JSON serializer, inputs containing duplicate map
// keys are rejected outright and not surfaced as a strict decoding error.
DupMapKey: cbor.DupMapKeyEnforcedAPF,
// For JSON parity, decoding an RFC3339 string into time.Time needs to be accepted
// with or without tagging. If a tag number is present, it must be valid.
TimeTag: cbor.DecTagOptional,
// Observed depth up to 16 in fuzzed batch/v1 CronJobList. JSON implementation limit
// is 10000.
MaxNestedLevels: 64,
MaxArrayElements: 1024,
MaxMapPairs: 1024,
// Indefinite-length sequences aren't produced by this serializer, but other
// implementations can.
IndefLength: cbor.IndefLengthAllowed,
// Accept inputs that contain CBOR tags.
TagsMd: cbor.TagsAllowed,
// Decode type 0 (unsigned integer) as int64.
// TODO: IntDecConvertSignedOrFail errors on overflow, JSON will try to fall back to float64.
IntDec: cbor.IntDecConvertSignedOrFail,
// Disable producing map[cbor.ByteString]interface{}, which is not acceptable for
// decodes into interface{}.
MapKeyByteString: cbor.MapKeyByteStringForbidden,
// Error on map keys that don't map to a field in the destination struct.
ExtraReturnErrors: cbor.ExtraDecErrorUnknownField,
// Decode maps into concrete type map[string]interface{} when the destination is an
// interface{}.
DefaultMapType: reflect.TypeOf(map[string]interface{}(nil)),
// A CBOR text string whose content is not a valid UTF-8 sequence is well-formed but
// invalid according to the CBOR spec. Reject invalid inputs. Encoders are
// responsible for ensuring that all text strings they produce contain valid UTF-8
// sequences and may use the byte string major type to encode strings that have not
// been validated.
UTF8: cbor.UTF8RejectInvalid,
// Never make a case-insensitive match between a map key and a struct field.
FieldNameMatching: cbor.FieldNameMatchingCaseSensitive,
// Produce string concrete values when decoding a CBOR byte string into interface{}.
DefaultByteStringType: reflect.TypeOf(""),
// Allow CBOR byte strings to be decoded into string destination values. If a byte
// string is enclosed in an "expected later encoding" tag
// (https://www.rfc-editor.org/rfc/rfc8949.html#section-3.4.5.2), then the text
// encoding indicated by that tag (e.g. base64) will be applied to the contents of
// the byte string.
ByteStringToString: cbor.ByteStringToStringAllowedWithExpectedLaterEncoding,
// Allow CBOR byte strings to match struct fields when appearing as a map key.
FieldNameByteString: cbor.FieldNameByteStringAllowed,
// When decoding an unrecognized tag to interface{}, return the decoded tag content
// instead of the default, a cbor.Tag representing a (number, content) pair.
UnrecognizedTagToAny: cbor.UnrecognizedTagContentToAny,
// Decode time tags to interface{} as strings containing RFC 3339 timestamps.
TimeTagToAny: cbor.TimeTagToRFC3339Nano,
// For parity with JSON, strings can be decoded into time.Time if they are RFC 3339
// timestamps.
ByteStringToTime: cbor.ByteStringToTimeAllowed,
// Reject NaN and infinite floating-point values since they don't have a JSON
// representation (RFC 8259 Section 6).
NaN: cbor.NaNDecodeForbidden,
Inf: cbor.InfDecodeForbidden,
// When unmarshaling a byte string into a []byte, assume that the byte string
// contains base64-encoded bytes, unless explicitly counterindicated by an "expected
// later encoding" tag. This is consistent with the because of unmarshaling a JSON
// text into a []byte.
ByteStringExpectedFormat: cbor.ByteStringExpectedBase64,
// Reject the arbitrary-precision integer tags because they can't be faithfully
// roundtripped through the allowable Unstructured types.
BignumTag: cbor.BignumTagForbidden,
// Reject anything other than the simple values true, false, and null.
SimpleValues: simpleValues,
// Disable default recognition of types implementing encoding.BinaryUnmarshaler,
// which is not recognized for JSON decoding.
BinaryUnmarshaler: cbor.BinaryUnmarshalerNone,
}.DecMode()
if err != nil {
panic(err)
}
return decode
}()
// DecodeLax is derived from Decode, but does not complain about unknown fields in the input.
var DecodeLax cbor.DecMode = func() cbor.DecMode {
opts := Decode.DecOptions()
opts.ExtraReturnErrors &^= cbor.ExtraDecErrorUnknownField // clear bit
dm, err := opts.DecMode()
if err != nil {
panic(err)
}
return dm
}()

View File

@ -0,0 +1,36 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package modes
import (
"github.com/fxamacker/cbor/v2"
)
var Diagnostic cbor.DiagMode = func() cbor.DiagMode {
opts := Decode.DecOptions()
diagnostic, err := cbor.DiagOptions{
ByteStringText: true,
MaxNestedLevels: opts.MaxNestedLevels,
MaxArrayElements: opts.MaxArrayElements,
MaxMapPairs: opts.MaxMapPairs,
}.DiagMode()
if err != nil {
panic(err)
}
return diagnostic
}()

View File

@ -0,0 +1,155 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package modes
import (
"io"
"github.com/fxamacker/cbor/v2"
)
var Encode = EncMode{
delegate: func() cbor.UserBufferEncMode {
encode, err := cbor.EncOptions{
// Map keys need to be sorted to have deterministic output, and this is the order
// defined in RFC 8949 4.2.1 "Core Deterministic Encoding Requirements".
Sort: cbor.SortBytewiseLexical,
// CBOR supports distinct types for IEEE-754 float16, float32, and float64. Store
// floats in the smallest width that preserves value so that equivalent float32 and
// float64 values encode to identical bytes, as they do in a JSON
// encoding. Satisfies one of the "Core Deterministic Encoding Requirements".
ShortestFloat: cbor.ShortestFloat16,
// Error on attempt to encode NaN and infinite values. This is what the JSON
// serializer does.
NaNConvert: cbor.NaNConvertReject,
InfConvert: cbor.InfConvertReject,
// Error on attempt to encode math/big.Int values, which can't be faithfully
// roundtripped through Unstructured in general (the dynamic numeric types allowed
// in Unstructured are limited to float64 and int64).
BigIntConvert: cbor.BigIntConvertReject,
// MarshalJSON for time.Time writes RFC3339 with nanos.
Time: cbor.TimeRFC3339Nano,
// The decoder must be able to accept RFC3339 strings with or without tag 0 (e.g. by
// the end of time.Time -> JSON -> Unstructured -> CBOR, the CBOR encoder has no
// reliable way of knowing that a particular string originated from serializing a
// time.Time), so producing tag 0 has little use.
TimeTag: cbor.EncTagNone,
// Indefinite-length items have multiple encodings and aren't being used anyway, so
// disable to avoid an opportunity for nondeterminism.
IndefLength: cbor.IndefLengthForbidden,
// Preserve distinction between nil and empty for slices and maps.
NilContainers: cbor.NilContainerAsNull,
// OK to produce tags.
TagsMd: cbor.TagsAllowed,
// Use the same definition of "empty" as encoding/json.
OmitEmpty: cbor.OmitEmptyGoValue,
// The CBOR types text string and byte string are structurally equivalent, with the
// semantic difference that a text string whose content is an invalid UTF-8 sequence
// is itself invalid. We reject all invalid text strings at decode time and do not
// validate or sanitize all Go strings at encode time. Encoding Go strings to the
// byte string type is comparable to the existing Protobuf behavior and cheaply
// ensures that the output is valid CBOR.
String: cbor.StringToByteString,
// Encode struct field names to the byte string type rather than the text string
// type.
FieldName: cbor.FieldNameToByteString,
// Marshal Go byte arrays to CBOR arrays of integers (as in JSON) instead of byte
// strings.
ByteArray: cbor.ByteArrayToArray,
// Marshal []byte to CBOR byte string enclosed in tag 22 (expected later base64
// encoding, https://www.rfc-editor.org/rfc/rfc8949.html#section-3.4.5.2), to
// interoperate with the existing JSON behavior. This indicates to the decoder that,
// when decoding into a string (or unstructured), the resulting value should be the
// base64 encoding of the original bytes. No base64 encoding or decoding needs to be
// performed for []byte-to-CBOR-to-[]byte roundtrips.
ByteSliceLaterFormat: cbor.ByteSliceLaterFormatBase64,
// Disable default recognition of types implementing encoding.BinaryMarshaler, which
// is not recognized for JSON encoding.
BinaryMarshaler: cbor.BinaryMarshalerNone,
}.UserBufferEncMode()
if err != nil {
panic(err)
}
return encode
}(),
}
var EncodeNondeterministic = EncMode{
delegate: func() cbor.UserBufferEncMode {
opts := Encode.options()
opts.Sort = cbor.SortFastShuffle
em, err := opts.UserBufferEncMode()
if err != nil {
panic(err)
}
return em
}(),
}
type EncMode struct {
delegate cbor.UserBufferEncMode
}
func (em EncMode) options() cbor.EncOptions {
return em.delegate.EncOptions()
}
func (em EncMode) MarshalTo(v interface{}, w io.Writer) error {
if buf, ok := w.(*buffer); ok {
return em.delegate.MarshalToBuffer(v, &buf.Buffer)
}
buf := buffers.Get()
defer buffers.Put(buf)
if err := em.delegate.MarshalToBuffer(v, &buf.Buffer); err != nil {
return err
}
if _, err := io.Copy(w, buf); err != nil {
return err
}
return nil
}
func (em EncMode) Marshal(v interface{}) ([]byte, error) {
buf := buffers.Get()
defer buffers.Put(buf)
if err := em.MarshalTo(v, &buf.Buffer); err != nil {
return nil, err
}
clone := make([]byte, buf.Len())
copy(clone, buf.Bytes())
return clone, nil
}

View File

@ -0,0 +1,236 @@
/*
Copyright 2024 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cbor
import (
"fmt"
"reflect"
"sync"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
var sharedTranscoders transcoders
var rawTypeTranscodeFuncs = map[reflect.Type]func(reflect.Value) error{
reflect.TypeFor[runtime.RawExtension](): func(rv reflect.Value) error {
if !rv.CanAddr() {
return nil
}
re := rv.Addr().Interface().(*runtime.RawExtension)
if re.Raw == nil {
// When Raw is nil it encodes to null. Don't change nil Raw values during
// transcoding, they would have unmarshalled from JSON as nil too.
return nil
}
j, err := re.MarshalJSON()
if err != nil {
return fmt.Errorf("failed to transcode RawExtension to JSON: %w", err)
}
re.Raw = j
return nil
},
reflect.TypeFor[metav1.FieldsV1](): func(rv reflect.Value) error {
if !rv.CanAddr() {
return nil
}
fields := rv.Addr().Interface().(*metav1.FieldsV1)
if fields.Raw == nil {
// When Raw is nil it encodes to null. Don't change nil Raw values during
// transcoding, they would have unmarshalled from JSON as nil too.
return nil
}
j, err := fields.MarshalJSON()
if err != nil {
return fmt.Errorf("failed to transcode FieldsV1 to JSON: %w", err)
}
fields.Raw = j
return nil
},
}
func transcodeRawTypes(v interface{}) error {
if v == nil {
return nil
}
rv := reflect.ValueOf(v)
return sharedTranscoders.getTranscoder(rv.Type()).fn(rv)
}
type transcoder struct {
fn func(rv reflect.Value) error
}
var noop = transcoder{
fn: func(reflect.Value) error {
return nil
},
}
type transcoders struct {
lock sync.RWMutex
m map[reflect.Type]**transcoder
}
func (ts *transcoders) getTranscoder(rt reflect.Type) transcoder {
ts.lock.RLock()
tpp, ok := ts.m[rt]
ts.lock.RUnlock()
if ok {
return **tpp
}
ts.lock.Lock()
defer ts.lock.Unlock()
tp := ts.getTranscoderLocked(rt)
return *tp
}
func (ts *transcoders) getTranscoderLocked(rt reflect.Type) *transcoder {
if tpp, ok := ts.m[rt]; ok {
// A transcoder for this type was cached while waiting to acquire the lock.
return *tpp
}
// Cache the transcoder now, before populating fn, so that circular references between types
// don't overflow the call stack.
t := new(transcoder)
if ts.m == nil {
ts.m = make(map[reflect.Type]**transcoder)
}
ts.m[rt] = &t
for rawType, fn := range rawTypeTranscodeFuncs {
if rt == rawType {
t = &transcoder{fn: fn}
return t
}
}
switch rt.Kind() {
case reflect.Array:
te := ts.getTranscoderLocked(rt.Elem())
rtlen := rt.Len()
if rtlen == 0 || te == &noop {
t = &noop
break
}
t.fn = func(rv reflect.Value) error {
for i := 0; i < rtlen; i++ {
if err := te.fn(rv.Index(i)); err != nil {
return err
}
}
return nil
}
case reflect.Interface:
// Any interface value might have a dynamic type involving RawExtension. It needs to
// be checked.
t.fn = func(rv reflect.Value) error {
if rv.IsNil() {
return nil
}
rv = rv.Elem()
// The interface element's type is dynamic so its transcoder can't be
// determined statically.
return ts.getTranscoder(rv.Type()).fn(rv)
}
case reflect.Map:
rtk := rt.Key()
tk := ts.getTranscoderLocked(rtk)
rte := rt.Elem()
te := ts.getTranscoderLocked(rte)
if tk == &noop && te == &noop {
t = &noop
break
}
t.fn = func(rv reflect.Value) error {
iter := rv.MapRange()
rvk := reflect.New(rtk).Elem()
rve := reflect.New(rte).Elem()
for iter.Next() {
rvk.SetIterKey(iter)
if err := tk.fn(rvk); err != nil {
return err
}
rve.SetIterValue(iter)
if err := te.fn(rve); err != nil {
return err
}
}
return nil
}
case reflect.Pointer:
te := ts.getTranscoderLocked(rt.Elem())
if te == &noop {
t = &noop
break
}
t.fn = func(rv reflect.Value) error {
if rv.IsNil() {
return nil
}
return te.fn(rv.Elem())
}
case reflect.Slice:
te := ts.getTranscoderLocked(rt.Elem())
if te == &noop {
t = &noop
break
}
t.fn = func(rv reflect.Value) error {
for i := 0; i < rv.Len(); i++ {
if err := te.fn(rv.Index(i)); err != nil {
return err
}
}
return nil
}
case reflect.Struct:
type fieldTranscoder struct {
Index int
Transcoder *transcoder
}
var fieldTranscoders []fieldTranscoder
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
tf := ts.getTranscoderLocked(f.Type)
if tf == &noop {
continue
}
fieldTranscoders = append(fieldTranscoders, fieldTranscoder{Index: i, Transcoder: tf})
}
if len(fieldTranscoders) == 0 {
t = &noop
break
}
t.fn = func(rv reflect.Value) error {
for _, ft := range fieldTranscoders {
if err := ft.Transcoder.fn(rv.Field(ft.Index)); err != nil {
return err
}
}
return nil
}
default:
t = &noop
}
return t
}

View File

@ -0,0 +1,305 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package serializer
import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/json"
"k8s.io/apimachinery/pkg/runtime/serializer/protobuf"
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
"k8s.io/apimachinery/pkg/runtime/serializer/versioning"
)
func newSerializersForScheme(scheme *runtime.Scheme, mf json.MetaFactory, options CodecFactoryOptions) []runtime.SerializerInfo {
jsonSerializer := json.NewSerializerWithOptions(
mf, scheme, scheme,
json.SerializerOptions{Yaml: false, Pretty: false, Strict: options.Strict},
)
jsonSerializerType := runtime.SerializerInfo{
MediaType: runtime.ContentTypeJSON,
MediaTypeType: "application",
MediaTypeSubType: "json",
EncodesAsText: true,
Serializer: jsonSerializer,
StrictSerializer: json.NewSerializerWithOptions(
mf, scheme, scheme,
json.SerializerOptions{Yaml: false, Pretty: false, Strict: true},
),
StreamSerializer: &runtime.StreamSerializerInfo{
EncodesAsText: true,
Serializer: jsonSerializer,
Framer: json.Framer,
},
}
if options.Pretty {
jsonSerializerType.PrettySerializer = json.NewSerializerWithOptions(
mf, scheme, scheme,
json.SerializerOptions{Yaml: false, Pretty: true, Strict: options.Strict},
)
}
yamlSerializer := json.NewSerializerWithOptions(
mf, scheme, scheme,
json.SerializerOptions{Yaml: true, Pretty: false, Strict: options.Strict},
)
strictYAMLSerializer := json.NewSerializerWithOptions(
mf, scheme, scheme,
json.SerializerOptions{Yaml: true, Pretty: false, Strict: true},
)
protoSerializer := protobuf.NewSerializer(scheme, scheme)
protoRawSerializer := protobuf.NewRawSerializer(scheme, scheme)
serializers := []runtime.SerializerInfo{
jsonSerializerType,
{
MediaType: runtime.ContentTypeYAML,
MediaTypeType: "application",
MediaTypeSubType: "yaml",
EncodesAsText: true,
Serializer: yamlSerializer,
StrictSerializer: strictYAMLSerializer,
},
{
MediaType: runtime.ContentTypeProtobuf,
MediaTypeType: "application",
MediaTypeSubType: "vnd.kubernetes.protobuf",
Serializer: protoSerializer,
// note, strict decoding is unsupported for protobuf,
// fall back to regular serializing
StrictSerializer: protoSerializer,
StreamSerializer: &runtime.StreamSerializerInfo{
Serializer: protoRawSerializer,
Framer: protobuf.LengthDelimitedFramer,
},
},
}
for _, f := range options.serializers {
serializers = append(serializers, f(scheme, scheme))
}
return serializers
}
// CodecFactory provides methods for retrieving codecs and serializers for specific
// versions and content types.
type CodecFactory struct {
scheme *runtime.Scheme
universal runtime.Decoder
accepts []runtime.SerializerInfo
legacySerializer runtime.Serializer
}
// CodecFactoryOptions holds the options for configuring CodecFactory behavior
type CodecFactoryOptions struct {
// Strict configures all serializers in strict mode
Strict bool
// Pretty includes a pretty serializer along with the non-pretty one
Pretty bool
serializers []func(runtime.ObjectCreater, runtime.ObjectTyper) runtime.SerializerInfo
}
// CodecFactoryOptionsMutator takes a pointer to an options struct and then modifies it.
// Functions implementing this type can be passed to the NewCodecFactory() constructor.
type CodecFactoryOptionsMutator func(*CodecFactoryOptions)
// EnablePretty enables including a pretty serializer along with the non-pretty one
func EnablePretty(options *CodecFactoryOptions) {
options.Pretty = true
}
// DisablePretty disables including a pretty serializer along with the non-pretty one
func DisablePretty(options *CodecFactoryOptions) {
options.Pretty = false
}
// EnableStrict enables configuring all serializers in strict mode
func EnableStrict(options *CodecFactoryOptions) {
options.Strict = true
}
// DisableStrict disables configuring all serializers in strict mode
func DisableStrict(options *CodecFactoryOptions) {
options.Strict = false
}
// WithSerializer configures a serializer to be supported in addition to the default serializers.
func WithSerializer(f func(runtime.ObjectCreater, runtime.ObjectTyper) runtime.SerializerInfo) CodecFactoryOptionsMutator {
return func(options *CodecFactoryOptions) {
options.serializers = append(options.serializers, f)
}
}
// NewCodecFactory provides methods for retrieving serializers for the supported wire formats
// and conversion wrappers to define preferred internal and external versions. In the future,
// as the internal version is used less, callers may instead use a defaulting serializer and
// only convert objects which are shared internally (Status, common API machinery).
//
// Mutators can be passed to change the CodecFactoryOptions before construction of the factory.
// It is recommended to explicitly pass mutators instead of relying on defaults.
// By default, Pretty is enabled -- this is conformant with previously supported behavior.
//
// TODO: allow other codecs to be compiled in?
// TODO: accept a scheme interface
func NewCodecFactory(scheme *runtime.Scheme, mutators ...CodecFactoryOptionsMutator) CodecFactory {
options := CodecFactoryOptions{Pretty: true}
for _, fn := range mutators {
fn(&options)
}
serializers := newSerializersForScheme(scheme, json.DefaultMetaFactory, options)
return newCodecFactory(scheme, serializers)
}
// newCodecFactory is a helper for testing that allows a different metafactory to be specified.
func newCodecFactory(scheme *runtime.Scheme, serializers []runtime.SerializerInfo) CodecFactory {
decoders := make([]runtime.Decoder, 0, len(serializers))
var accepts []runtime.SerializerInfo
alreadyAccepted := make(map[string]struct{})
var legacySerializer runtime.Serializer
for _, d := range serializers {
decoders = append(decoders, d.Serializer)
if _, ok := alreadyAccepted[d.MediaType]; ok {
continue
}
alreadyAccepted[d.MediaType] = struct{}{}
acceptedSerializerShallowCopy := d
if d.StreamSerializer != nil {
cloned := *d.StreamSerializer
acceptedSerializerShallowCopy.StreamSerializer = &cloned
}
accepts = append(accepts, acceptedSerializerShallowCopy)
if d.MediaType == runtime.ContentTypeJSON {
legacySerializer = d.Serializer
}
}
if legacySerializer == nil {
legacySerializer = serializers[0].Serializer
}
return CodecFactory{
scheme: scheme,
universal: recognizer.NewDecoder(decoders...),
accepts: accepts,
legacySerializer: legacySerializer,
}
}
// WithoutConversion returns a NegotiatedSerializer that performs no conversion, even if the
// caller requests it.
func (f CodecFactory) WithoutConversion() runtime.NegotiatedSerializer {
return WithoutConversionCodecFactory{f}
}
// SupportedMediaTypes returns the RFC2046 media types that this factory has serializers for.
func (f CodecFactory) SupportedMediaTypes() []runtime.SerializerInfo {
return f.accepts
}
// LegacyCodec encodes output to a given API versions, and decodes output into the internal form from
// any recognized source. The returned codec will always encode output to JSON. If a type is not
// found in the list of versions an error will be returned.
//
// This method is deprecated - clients and servers should negotiate a serializer by mime-type and
// invoke CodecForVersions. Callers that need only to read data should use UniversalDecoder().
//
// TODO: make this call exist only in pkg/api, and initialize it with the set of default versions.
// All other callers will be forced to request a Codec directly.
func (f CodecFactory) LegacyCodec(version ...schema.GroupVersion) runtime.Codec {
return versioning.NewDefaultingCodecForScheme(f.scheme, f.legacySerializer, f.universal, schema.GroupVersions(version), runtime.InternalGroupVersioner)
}
// UniversalDeserializer can convert any stored data recognized by this factory into a Go object that satisfies
// runtime.Object. It does not perform conversion. It does not perform defaulting.
func (f CodecFactory) UniversalDeserializer() runtime.Decoder {
return f.universal
}
// UniversalDecoder returns a runtime.Decoder capable of decoding all known API objects in all known formats. Used
// by clients that do not need to encode objects but want to deserialize API objects stored on disk. Only decodes
// objects in groups registered with the scheme. The GroupVersions passed may be used to select alternate
// versions of objects to return - by default, runtime.APIVersionInternal is used. If any versions are specified,
// unrecognized groups will be returned in the version they are encoded as (no conversion). This decoder performs
// defaulting.
//
// TODO: the decoder will eventually be removed in favor of dealing with objects in their versioned form
// TODO: only accept a group versioner
func (f CodecFactory) UniversalDecoder(versions ...schema.GroupVersion) runtime.Decoder {
var versioner runtime.GroupVersioner
if len(versions) == 0 {
versioner = runtime.InternalGroupVersioner
} else {
versioner = schema.GroupVersions(versions)
}
return f.CodecForVersions(nil, f.universal, nil, versioner)
}
// CodecForVersions creates a codec with the provided serializer. If an object is decoded and its group is not in the list,
// it will default to runtime.APIVersionInternal. If encode is not specified for an object's group, the object is not
// converted. If encode or decode are nil, no conversion is performed.
func (f CodecFactory) CodecForVersions(encoder runtime.Encoder, decoder runtime.Decoder, encode runtime.GroupVersioner, decode runtime.GroupVersioner) runtime.Codec {
// TODO: these are for backcompat, remove them in the future
if encode == nil {
encode = runtime.DisabledGroupVersioner
}
if decode == nil {
decode = runtime.InternalGroupVersioner
}
return versioning.NewDefaultingCodecForScheme(f.scheme, encoder, decoder, encode, decode)
}
// DecoderToVersion returns a decoder that targets the provided group version.
func (f CodecFactory) DecoderToVersion(decoder runtime.Decoder, gv runtime.GroupVersioner) runtime.Decoder {
return f.CodecForVersions(nil, decoder, nil, gv)
}
// EncoderForVersion returns an encoder that targets the provided group version.
func (f CodecFactory) EncoderForVersion(encoder runtime.Encoder, gv runtime.GroupVersioner) runtime.Encoder {
return f.CodecForVersions(encoder, nil, gv, nil)
}
// WithoutConversionCodecFactory is a CodecFactory that will explicitly ignore requests to perform conversion.
// This wrapper is used while code migrates away from using conversion (such as external clients) and in the future
// will be unnecessary when we change the signature of NegotiatedSerializer.
type WithoutConversionCodecFactory struct {
CodecFactory
}
// EncoderForVersion returns an encoder that does not do conversion, but does set the group version kind of the object
// when serialized.
func (f WithoutConversionCodecFactory) EncoderForVersion(serializer runtime.Encoder, version runtime.GroupVersioner) runtime.Encoder {
return runtime.WithVersionEncoder{
Version: version,
Encoder: serializer,
ObjectTyper: f.CodecFactory.scheme,
}
}
// DecoderToVersion returns an decoder that does not do conversion.
func (f WithoutConversionCodecFactory) DecoderToVersion(serializer runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder {
return runtime.WithoutVersionDecoder{
Decoder: serializer,
}
}

View File

@ -0,0 +1,351 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package json
import (
"encoding/json"
"io"
"strconv"
kjson "sigs.k8s.io/json"
"sigs.k8s.io/yaml"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
"k8s.io/apimachinery/pkg/util/framer"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/klog/v2"
)
// NewSerializer creates a JSON serializer that handles encoding versioned objects into the proper JSON form. If typer
// is not nil, the object has the group, version, and kind fields set.
// Deprecated: use NewSerializerWithOptions instead.
func NewSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, pretty bool) *Serializer {
return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{false, pretty, false})
}
// NewYAMLSerializer creates a YAML serializer that handles encoding versioned objects into the proper YAML form. If typer
// is not nil, the object has the group, version, and kind fields set. This serializer supports only the subset of YAML that
// matches JSON, and will error if constructs are used that do not serialize to JSON.
// Deprecated: use NewSerializerWithOptions instead.
func NewYAMLSerializer(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper) *Serializer {
return NewSerializerWithOptions(meta, creater, typer, SerializerOptions{true, false, false})
}
// NewSerializerWithOptions creates a JSON/YAML serializer that handles encoding versioned objects into the proper JSON/YAML
// form. If typer is not nil, the object has the group, version, and kind fields set. Options are copied into the Serializer
// and are immutable.
func NewSerializerWithOptions(meta MetaFactory, creater runtime.ObjectCreater, typer runtime.ObjectTyper, options SerializerOptions) *Serializer {
return &Serializer{
meta: meta,
creater: creater,
typer: typer,
options: options,
identifier: identifier(options),
}
}
// identifier computes Identifier of Encoder based on the given options.
func identifier(options SerializerOptions) runtime.Identifier {
result := map[string]string{
"name": "json",
"yaml": strconv.FormatBool(options.Yaml),
"pretty": strconv.FormatBool(options.Pretty),
"strict": strconv.FormatBool(options.Strict),
}
identifier, err := json.Marshal(result)
if err != nil {
klog.Fatalf("Failed marshaling identifier for json Serializer: %v", err)
}
return runtime.Identifier(identifier)
}
// SerializerOptions holds the options which are used to configure a JSON/YAML serializer.
// example:
// (1) To configure a JSON serializer, set `Yaml` to `false`.
// (2) To configure a YAML serializer, set `Yaml` to `true`.
// (3) To configure a strict serializer that can return strictDecodingError, set `Strict` to `true`.
type SerializerOptions struct {
// Yaml: configures the Serializer to work with JSON(false) or YAML(true).
// When `Yaml` is enabled, this serializer only supports the subset of YAML that
// matches JSON, and will error if constructs are used that do not serialize to JSON.
Yaml bool
// Pretty: configures a JSON enabled Serializer(`Yaml: false`) to produce human-readable output.
// This option is silently ignored when `Yaml` is `true`.
Pretty bool
// Strict: configures the Serializer to return strictDecodingError's when duplicate fields are present decoding JSON or YAML.
// Note that enabling this option is not as performant as the non-strict variant, and should not be used in fast paths.
Strict bool
}
// Serializer handles encoding versioned objects into the proper JSON form
type Serializer struct {
meta MetaFactory
options SerializerOptions
creater runtime.ObjectCreater
typer runtime.ObjectTyper
identifier runtime.Identifier
}
// Serializer implements Serializer
var _ runtime.Serializer = &Serializer{}
var _ recognizer.RecognizingDecoder = &Serializer{}
// gvkWithDefaults returns group kind and version defaulting from provided default
func gvkWithDefaults(actual, defaultGVK schema.GroupVersionKind) schema.GroupVersionKind {
if len(actual.Kind) == 0 {
actual.Kind = defaultGVK.Kind
}
if len(actual.Version) == 0 && len(actual.Group) == 0 {
actual.Group = defaultGVK.Group
actual.Version = defaultGVK.Version
}
if len(actual.Version) == 0 && actual.Group == defaultGVK.Group {
actual.Version = defaultGVK.Version
}
return actual
}
// Decode attempts to convert the provided data into YAML or JSON, extract the stored schema kind, apply the provided default gvk, and then
// load that data into an object matching the desired schema kind or the provided into.
// If into is *runtime.Unknown, the raw data will be extracted and no decoding will be performed.
// If into is not registered with the typer, then the object will be straight decoded using normal JSON/YAML unmarshalling.
// If into is provided and the original data is not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk.
// If into is nil or data's gvk different from into's gvk, it will generate a new Object with ObjectCreater.New(gvk)
// On success or most errors, the method will return the calculated schema kind.
// The gvk calculate priority will be originalData > default gvk > into
func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
data := originalData
if s.options.Yaml {
altered, err := yaml.YAMLToJSON(data)
if err != nil {
return nil, nil, err
}
data = altered
}
actual, err := s.meta.Interpret(data)
if err != nil {
return nil, nil, err
}
if gvk != nil {
*actual = gvkWithDefaults(*actual, *gvk)
}
if unk, ok := into.(*runtime.Unknown); ok && unk != nil {
unk.Raw = originalData
unk.ContentType = runtime.ContentTypeJSON
unk.GetObjectKind().SetGroupVersionKind(*actual)
return unk, actual, nil
}
if into != nil {
_, isUnstructured := into.(runtime.Unstructured)
types, _, err := s.typer.ObjectKinds(into)
switch {
case runtime.IsNotRegisteredError(err), isUnstructured:
strictErrs, err := s.unmarshal(into, data, originalData)
if err != nil {
return nil, actual, err
}
// when decoding directly into a provided unstructured object,
// extract the actual gvk decoded from the provided data,
// and ensure it is non-empty.
if isUnstructured {
*actual = into.GetObjectKind().GroupVersionKind()
if len(actual.Kind) == 0 {
return nil, actual, runtime.NewMissingKindErr(string(originalData))
}
// TODO(109023): require apiVersion here as well once unstructuredJSONScheme#Decode does
}
if len(strictErrs) > 0 {
return into, actual, runtime.NewStrictDecodingError(strictErrs)
}
return into, actual, nil
case err != nil:
return nil, actual, err
default:
*actual = gvkWithDefaults(*actual, types[0])
}
}
if len(actual.Kind) == 0 {
return nil, actual, runtime.NewMissingKindErr(string(originalData))
}
if len(actual.Version) == 0 {
return nil, actual, runtime.NewMissingVersionErr(string(originalData))
}
// use the target if necessary
obj, err := runtime.UseOrCreateObject(s.typer, s.creater, *actual, into)
if err != nil {
return nil, actual, err
}
strictErrs, err := s.unmarshal(obj, data, originalData)
if err != nil {
return nil, actual, err
} else if len(strictErrs) > 0 {
return obj, actual, runtime.NewStrictDecodingError(strictErrs)
}
return obj, actual, nil
}
// Encode serializes the provided object to the given writer.
func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
if co, ok := obj.(runtime.CacheableObject); ok {
return co.CacheEncode(s.Identifier(), s.doEncode, w)
}
return s.doEncode(obj, w)
}
func (s *Serializer) doEncode(obj runtime.Object, w io.Writer) error {
if s.options.Yaml {
json, err := json.Marshal(obj)
if err != nil {
return err
}
data, err := yaml.JSONToYAML(json)
if err != nil {
return err
}
_, err = w.Write(data)
return err
}
if s.options.Pretty {
data, err := json.MarshalIndent(obj, "", " ")
if err != nil {
return err
}
_, err = w.Write(data)
return err
}
encoder := json.NewEncoder(w)
return encoder.Encode(obj)
}
// IsStrict indicates whether the serializer
// uses strict decoding or not
func (s *Serializer) IsStrict() bool {
return s.options.Strict
}
func (s *Serializer) unmarshal(into runtime.Object, data, originalData []byte) (strictErrs []error, err error) {
// If the deserializer is non-strict, return here.
if !s.options.Strict {
if err := kjson.UnmarshalCaseSensitivePreserveInts(data, into); err != nil {
return nil, err
}
return nil, nil
}
if s.options.Yaml {
// In strict mode pass the original data through the YAMLToJSONStrict converter.
// This is done to catch duplicate fields in YAML that would have been dropped in the original YAMLToJSON conversion.
// TODO: rework YAMLToJSONStrict to return warnings about duplicate fields without terminating so we don't have to do this twice.
_, err := yaml.YAMLToJSONStrict(originalData)
if err != nil {
strictErrs = append(strictErrs, err)
}
}
var strictJSONErrs []error
if u, isUnstructured := into.(runtime.Unstructured); isUnstructured {
// Unstructured is a custom unmarshaler that gets delegated
// to, so in order to detect strict JSON errors we need
// to unmarshal directly into the object.
m := map[string]interface{}{}
strictJSONErrs, err = kjson.UnmarshalStrict(data, &m)
u.SetUnstructuredContent(m)
} else {
strictJSONErrs, err = kjson.UnmarshalStrict(data, into)
}
if err != nil {
// fatal decoding error, not due to strictness
return nil, err
}
strictErrs = append(strictErrs, strictJSONErrs...)
return strictErrs, nil
}
// Identifier implements runtime.Encoder interface.
func (s *Serializer) Identifier() runtime.Identifier {
return s.identifier
}
// RecognizesData implements the RecognizingDecoder interface.
func (s *Serializer) RecognizesData(data []byte) (ok, unknown bool, err error) {
if s.options.Yaml {
// we could potentially look for '---'
return false, true, nil
}
return utilyaml.IsJSONBuffer(data), false, nil
}
// Framer is the default JSON framing behavior, with newlines delimiting individual objects.
var Framer = jsonFramer{}
type jsonFramer struct{}
// NewFrameWriter implements stream framing for this serializer
func (jsonFramer) NewFrameWriter(w io.Writer) io.Writer {
// we can write JSON objects directly to the writer, because they are self-framing
return w
}
// NewFrameReader implements stream framing for this serializer
func (jsonFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser {
// we need to extract the JSON chunks of data to pass to Decode()
return framer.NewJSONFramedReader(r)
}
// YAMLFramer is the default JSON framing behavior, with newlines delimiting individual objects.
var YAMLFramer = yamlFramer{}
type yamlFramer struct{}
// NewFrameWriter implements stream framing for this serializer
func (yamlFramer) NewFrameWriter(w io.Writer) io.Writer {
return yamlFrameWriter{w}
}
// NewFrameReader implements stream framing for this serializer
func (yamlFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser {
// extract the YAML document chunks directly
return utilyaml.NewDocumentDecoder(r)
}
type yamlFrameWriter struct {
w io.Writer
}
// Write separates each document with the YAML document separator (`---` followed by line
// break). Writers must write well formed YAML documents (include a final line break).
func (w yamlFrameWriter) Write(data []byte) (n int, err error) {
if _, err := w.w.Write([]byte("---\n")); err != nil {
return 0, err
}
return w.w.Write(data)
}

View File

@ -0,0 +1,63 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package json
import (
"encoding/json"
"fmt"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// MetaFactory is used to store and retrieve the version and kind
// information for JSON objects in a serializer.
type MetaFactory interface {
// Interpret should return the version and kind of the wire-format of
// the object.
Interpret(data []byte) (*schema.GroupVersionKind, error)
}
// DefaultMetaFactory is a default factory for versioning objects in JSON. The object
// in memory and in the default JSON serialization will use the "kind" and "apiVersion"
// fields.
var DefaultMetaFactory = SimpleMetaFactory{}
// SimpleMetaFactory provides default methods for retrieving the type and version of objects
// that are identified with an "apiVersion" and "kind" fields in their JSON
// serialization. It may be parameterized with the names of the fields in memory, or an
// optional list of base structs to search for those fields in memory.
type SimpleMetaFactory struct {
}
// Interpret will return the APIVersion and Kind of the JSON wire-format
// encoding of an object, or an error.
func (SimpleMetaFactory) Interpret(data []byte) (*schema.GroupVersionKind, error) {
findKind := struct {
// +optional
APIVersion string `json:"apiVersion,omitempty"`
// +optional
Kind string `json:"kind,omitempty"`
}{}
if err := json.Unmarshal(data, &findKind); err != nil {
return nil, fmt.Errorf("couldn't get version/kind; json parse error: %v", err)
}
gv, err := schema.ParseGroupVersion(findKind.APIVersion)
if err != nil {
return nil, err
}
return &schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: findKind.Kind}, nil
}

View File

@ -0,0 +1,43 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package serializer
import (
"k8s.io/apimachinery/pkg/runtime"
)
// TODO: We should split negotiated serializers that we can change versions on from those we can change
// serialization formats on
type negotiatedSerializerWrapper struct {
info runtime.SerializerInfo
}
func NegotiatedSerializerWrapper(info runtime.SerializerInfo) runtime.NegotiatedSerializer {
return &negotiatedSerializerWrapper{info}
}
func (n *negotiatedSerializerWrapper) SupportedMediaTypes() []runtime.SerializerInfo {
return []runtime.SerializerInfo{n.info}
}
func (n *negotiatedSerializerWrapper) EncoderForVersion(e runtime.Encoder, _ runtime.GroupVersioner) runtime.Encoder {
return e
}
func (n *negotiatedSerializerWrapper) DecoderToVersion(d runtime.Decoder, _gv runtime.GroupVersioner) runtime.Decoder {
return d
}

View File

@ -0,0 +1,18 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package protobuf provides a Kubernetes serializer for the protobuf format.
package protobuf // import "k8s.io/apimachinery/pkg/runtime/serializer/protobuf"

View File

@ -0,0 +1,495 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package protobuf
import (
"bytes"
"fmt"
"io"
"net/http"
"reflect"
"github.com/gogo/protobuf/proto"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer/recognizer"
"k8s.io/apimachinery/pkg/util/framer"
"k8s.io/klog/v2"
)
var (
// protoEncodingPrefix serves as a magic number for an encoded protobuf message on this serializer. All
// proto messages serialized by this schema will be preceded by the bytes 0x6b 0x38 0x73, with the fourth
// byte being reserved for the encoding style. The only encoding style defined is 0x00, which means that
// the rest of the byte stream is a message of type k8s.io.kubernetes.pkg.runtime.Unknown (proto2).
//
// See k8s.io/apimachinery/pkg/runtime/generated.proto for details of the runtime.Unknown message.
//
// This encoding scheme is experimental, and is subject to change at any time.
protoEncodingPrefix = []byte{0x6b, 0x38, 0x73, 0x00}
)
type errNotMarshalable struct {
t reflect.Type
}
func (e errNotMarshalable) Error() string {
return fmt.Sprintf("object %v does not implement the protobuf marshalling interface and cannot be encoded to a protobuf message", e.t)
}
func (e errNotMarshalable) Status() metav1.Status {
return metav1.Status{
Status: metav1.StatusFailure,
Code: http.StatusNotAcceptable,
Reason: metav1.StatusReason("NotAcceptable"),
Message: e.Error(),
}
}
// IsNotMarshalable checks the type of error, returns a boolean true if error is not nil and not marshalable false otherwise
func IsNotMarshalable(err error) bool {
_, ok := err.(errNotMarshalable)
return err != nil && ok
}
// NewSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If a typer
// is passed, the encoded object will have group, version, and kind fields set. If typer is nil, the objects will be written
// as-is (any type info passed with the object will be used).
func NewSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper) *Serializer {
return &Serializer{
prefix: protoEncodingPrefix,
creater: creater,
typer: typer,
}
}
// Serializer handles encoding versioned objects into the proper wire form
type Serializer struct {
prefix []byte
creater runtime.ObjectCreater
typer runtime.ObjectTyper
}
var _ runtime.Serializer = &Serializer{}
var _ runtime.EncoderWithAllocator = &Serializer{}
var _ recognizer.RecognizingDecoder = &Serializer{}
const serializerIdentifier runtime.Identifier = "protobuf"
// Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default
// gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown,
// the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will
// be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is
// not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most
// errors, the method will return the calculated schema kind.
func (s *Serializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
prefixLen := len(s.prefix)
switch {
case len(originalData) == 0:
// TODO: treat like decoding {} from JSON with defaulting
return nil, nil, fmt.Errorf("empty data")
case len(originalData) < prefixLen || !bytes.Equal(s.prefix, originalData[:prefixLen]):
return nil, nil, fmt.Errorf("provided data does not appear to be a protobuf message, expected prefix %v", s.prefix)
case len(originalData) == prefixLen:
// TODO: treat like decoding {} from JSON with defaulting
return nil, nil, fmt.Errorf("empty body")
}
data := originalData[prefixLen:]
unk := runtime.Unknown{}
if err := unk.Unmarshal(data); err != nil {
return nil, nil, err
}
actual := unk.GroupVersionKind()
copyKindDefaults(&actual, gvk)
if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil {
*intoUnknown = unk
if ok, _, _ := s.RecognizesData(unk.Raw); ok {
intoUnknown.ContentType = runtime.ContentTypeProtobuf
}
return intoUnknown, &actual, nil
}
if into != nil {
types, _, err := s.typer.ObjectKinds(into)
switch {
case runtime.IsNotRegisteredError(err):
pb, ok := into.(proto.Message)
if !ok {
return nil, &actual, errNotMarshalable{reflect.TypeOf(into)}
}
if err := proto.Unmarshal(unk.Raw, pb); err != nil {
return nil, &actual, err
}
return into, &actual, nil
case err != nil:
return nil, &actual, err
default:
copyKindDefaults(&actual, &types[0])
// if the result of defaulting did not set a version or group, ensure that at least group is set
// (copyKindDefaults will not assign Group if version is already set). This guarantees that the group
// of into is set if there is no better information from the caller or object.
if len(actual.Version) == 0 && len(actual.Group) == 0 {
actual.Group = types[0].Group
}
}
}
if len(actual.Kind) == 0 {
return nil, &actual, runtime.NewMissingKindErr(fmt.Sprintf("%#v", unk.TypeMeta))
}
if len(actual.Version) == 0 {
return nil, &actual, runtime.NewMissingVersionErr(fmt.Sprintf("%#v", unk.TypeMeta))
}
return unmarshalToObject(s.typer, s.creater, &actual, into, unk.Raw)
}
// EncodeWithAllocator writes an object to the provided writer.
// In addition, it allows for providing a memory allocator for efficient memory usage during object serialization.
func (s *Serializer) EncodeWithAllocator(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
return s.encode(obj, w, memAlloc)
}
// Encode serializes the provided object to the given writer.
func (s *Serializer) Encode(obj runtime.Object, w io.Writer) error {
return s.encode(obj, w, &runtime.SimpleAllocator{})
}
func (s *Serializer) encode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
if co, ok := obj.(runtime.CacheableObject); ok {
return co.CacheEncode(s.Identifier(), func(obj runtime.Object, w io.Writer) error { return s.doEncode(obj, w, memAlloc) }, w)
}
return s.doEncode(obj, w, memAlloc)
}
func (s *Serializer) doEncode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
if memAlloc == nil {
klog.Error("a mandatory memory allocator wasn't provided, this might have a negative impact on performance, check invocations of EncodeWithAllocator method, falling back on runtime.SimpleAllocator")
memAlloc = &runtime.SimpleAllocator{}
}
prefixSize := uint64(len(s.prefix))
var unk runtime.Unknown
switch t := obj.(type) {
case *runtime.Unknown:
estimatedSize := prefixSize + uint64(t.Size())
data := memAlloc.Allocate(estimatedSize)
i, err := t.MarshalTo(data[prefixSize:])
if err != nil {
return err
}
copy(data, s.prefix)
_, err = w.Write(data[:prefixSize+uint64(i)])
return err
default:
kind := obj.GetObjectKind().GroupVersionKind()
unk = runtime.Unknown{
TypeMeta: runtime.TypeMeta{
Kind: kind.Kind,
APIVersion: kind.GroupVersion().String(),
},
}
}
switch t := obj.(type) {
case bufferedMarshaller:
// this path performs a single allocation during write only when the Allocator wasn't provided
// it also requires the caller to implement the more efficient Size and MarshalToSizedBuffer methods
encodedSize := uint64(t.Size())
estimatedSize := prefixSize + estimateUnknownSize(&unk, encodedSize)
data := memAlloc.Allocate(estimatedSize)
i, err := unk.NestedMarshalTo(data[prefixSize:], t, encodedSize)
if err != nil {
return err
}
copy(data, s.prefix)
_, err = w.Write(data[:prefixSize+uint64(i)])
return err
case proto.Marshaler:
// this path performs extra allocations
data, err := t.Marshal()
if err != nil {
return err
}
unk.Raw = data
estimatedSize := prefixSize + uint64(unk.Size())
data = memAlloc.Allocate(estimatedSize)
i, err := unk.MarshalTo(data[prefixSize:])
if err != nil {
return err
}
copy(data, s.prefix)
_, err = w.Write(data[:prefixSize+uint64(i)])
return err
default:
// TODO: marshal with a different content type and serializer (JSON for third party objects)
return errNotMarshalable{reflect.TypeOf(obj)}
}
}
// Identifier implements runtime.Encoder interface.
func (s *Serializer) Identifier() runtime.Identifier {
return serializerIdentifier
}
// RecognizesData implements the RecognizingDecoder interface.
func (s *Serializer) RecognizesData(data []byte) (bool, bool, error) {
return bytes.HasPrefix(data, s.prefix), false, nil
}
// copyKindDefaults defaults dst to the value in src if dst does not have a value set.
func copyKindDefaults(dst, src *schema.GroupVersionKind) {
if src == nil {
return
}
// apply kind and version defaulting from provided default
if len(dst.Kind) == 0 {
dst.Kind = src.Kind
}
if len(dst.Version) == 0 && len(src.Version) > 0 {
dst.Group = src.Group
dst.Version = src.Version
}
}
// bufferedMarshaller describes a more efficient marshalling interface that can avoid allocating multiple
// byte buffers by pre-calculating the size of the final buffer needed.
type bufferedMarshaller interface {
proto.Sizer
runtime.ProtobufMarshaller
}
// Like bufferedMarshaller, but is able to marshal backwards, which is more efficient since it doesn't call Size() as frequently.
type bufferedReverseMarshaller interface {
proto.Sizer
runtime.ProtobufReverseMarshaller
}
// estimateUnknownSize returns the expected bytes consumed by a given runtime.Unknown
// object with a nil RawJSON struct and the expected size of the provided buffer. The
// returned size will not be correct if RawJSOn is set on unk.
func estimateUnknownSize(unk *runtime.Unknown, byteSize uint64) uint64 {
size := uint64(unk.Size())
// protobuf uses 1 byte for the tag, a varint for the length of the array (at most 8 bytes - uint64 - here),
// and the size of the array.
size += 1 + 8 + byteSize
return size
}
// NewRawSerializer creates a Protobuf serializer that handles encoding versioned objects into the proper wire form. If typer
// is not nil, the object has the group, version, and kind fields set. This serializer does not provide type information for the
// encoded object, and thus is not self describing (callers must know what type is being described in order to decode).
//
// This encoding scheme is experimental, and is subject to change at any time.
func NewRawSerializer(creater runtime.ObjectCreater, typer runtime.ObjectTyper) *RawSerializer {
return &RawSerializer{
creater: creater,
typer: typer,
}
}
// RawSerializer encodes and decodes objects without adding a runtime.Unknown wrapper (objects are encoded without identifying
// type).
type RawSerializer struct {
creater runtime.ObjectCreater
typer runtime.ObjectTyper
}
var _ runtime.Serializer = &RawSerializer{}
const rawSerializerIdentifier runtime.Identifier = "raw-protobuf"
// Decode attempts to convert the provided data into a protobuf message, extract the stored schema kind, apply the provided default
// gvk, and then load that data into an object matching the desired schema kind or the provided into. If into is *runtime.Unknown,
// the raw data will be extracted and no decoding will be performed. If into is not registered with the typer, then the object will
// be straight decoded using normal protobuf unmarshalling (the MarshalTo interface). If into is provided and the original data is
// not fully qualified with kind/version/group, the type of the into will be used to alter the returned gvk. On success or most
// errors, the method will return the calculated schema kind.
func (s *RawSerializer) Decode(originalData []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
if into == nil {
return nil, nil, fmt.Errorf("this serializer requires an object to decode into: %#v", s)
}
if len(originalData) == 0 {
// TODO: treat like decoding {} from JSON with defaulting
return nil, nil, fmt.Errorf("empty data")
}
data := originalData
actual := &schema.GroupVersionKind{}
copyKindDefaults(actual, gvk)
if intoUnknown, ok := into.(*runtime.Unknown); ok && intoUnknown != nil {
intoUnknown.Raw = data
intoUnknown.ContentEncoding = ""
intoUnknown.ContentType = runtime.ContentTypeProtobuf
intoUnknown.SetGroupVersionKind(*actual)
return intoUnknown, actual, nil
}
types, _, err := s.typer.ObjectKinds(into)
switch {
case runtime.IsNotRegisteredError(err):
pb, ok := into.(proto.Message)
if !ok {
return nil, actual, errNotMarshalable{reflect.TypeOf(into)}
}
if err := proto.Unmarshal(data, pb); err != nil {
return nil, actual, err
}
return into, actual, nil
case err != nil:
return nil, actual, err
default:
copyKindDefaults(actual, &types[0])
// if the result of defaulting did not set a version or group, ensure that at least group is set
// (copyKindDefaults will not assign Group if version is already set). This guarantees that the group
// of into is set if there is no better information from the caller or object.
if len(actual.Version) == 0 && len(actual.Group) == 0 {
actual.Group = types[0].Group
}
}
if len(actual.Kind) == 0 {
return nil, actual, runtime.NewMissingKindErr("<protobuf encoded body - must provide default type>")
}
if len(actual.Version) == 0 {
return nil, actual, runtime.NewMissingVersionErr("<protobuf encoded body - must provide default type>")
}
return unmarshalToObject(s.typer, s.creater, actual, into, data)
}
// unmarshalToObject is the common code between decode in the raw and normal serializer.
func unmarshalToObject(typer runtime.ObjectTyper, creater runtime.ObjectCreater, actual *schema.GroupVersionKind, into runtime.Object, data []byte) (runtime.Object, *schema.GroupVersionKind, error) {
// use the target if necessary
obj, err := runtime.UseOrCreateObject(typer, creater, *actual, into)
if err != nil {
return nil, actual, err
}
pb, ok := obj.(proto.Message)
if !ok {
return nil, actual, errNotMarshalable{reflect.TypeOf(obj)}
}
if err := proto.Unmarshal(data, pb); err != nil {
return nil, actual, err
}
if actual != nil {
obj.GetObjectKind().SetGroupVersionKind(*actual)
}
return obj, actual, nil
}
// Encode serializes the provided object to the given writer. Overrides is ignored.
func (s *RawSerializer) Encode(obj runtime.Object, w io.Writer) error {
return s.encode(obj, w, &runtime.SimpleAllocator{})
}
// EncodeWithAllocator writes an object to the provided writer.
// In addition, it allows for providing a memory allocator for efficient memory usage during object serialization.
func (s *RawSerializer) EncodeWithAllocator(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
return s.encode(obj, w, memAlloc)
}
func (s *RawSerializer) encode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
if co, ok := obj.(runtime.CacheableObject); ok {
return co.CacheEncode(s.Identifier(), func(obj runtime.Object, w io.Writer) error { return s.doEncode(obj, w, memAlloc) }, w)
}
return s.doEncode(obj, w, memAlloc)
}
func (s *RawSerializer) doEncode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
if memAlloc == nil {
klog.Error("a mandatory memory allocator wasn't provided, this might have a negative impact on performance, check invocations of EncodeWithAllocator method, falling back on runtime.SimpleAllocator")
memAlloc = &runtime.SimpleAllocator{}
}
switch t := obj.(type) {
case bufferedReverseMarshaller:
// this path performs a single allocation during write only when the Allocator wasn't provided
// it also requires the caller to implement the more efficient Size and MarshalToSizedBuffer methods
encodedSize := uint64(t.Size())
data := memAlloc.Allocate(encodedSize)
n, err := t.MarshalToSizedBuffer(data)
if err != nil {
return err
}
_, err = w.Write(data[:n])
return err
case bufferedMarshaller:
// this path performs a single allocation during write only when the Allocator wasn't provided
// it also requires the caller to implement the more efficient Size and MarshalTo methods
encodedSize := uint64(t.Size())
data := memAlloc.Allocate(encodedSize)
n, err := t.MarshalTo(data)
if err != nil {
return err
}
_, err = w.Write(data[:n])
return err
case proto.Marshaler:
// this path performs extra allocations
data, err := t.Marshal()
if err != nil {
return err
}
_, err = w.Write(data)
return err
default:
return errNotMarshalable{reflect.TypeOf(obj)}
}
}
// Identifier implements runtime.Encoder interface.
func (s *RawSerializer) Identifier() runtime.Identifier {
return rawSerializerIdentifier
}
// LengthDelimitedFramer is exported variable of type lengthDelimitedFramer
var LengthDelimitedFramer = lengthDelimitedFramer{}
// Provides length delimited frame reader and writer methods
type lengthDelimitedFramer struct{}
// NewFrameWriter implements stream framing for this serializer
func (lengthDelimitedFramer) NewFrameWriter(w io.Writer) io.Writer {
return framer.NewLengthDelimitedFrameWriter(w)
}
// NewFrameReader implements stream framing for this serializer
func (lengthDelimitedFramer) NewFrameReader(r io.ReadCloser) io.ReadCloser {
return framer.NewLengthDelimitedFrameReader(r)
}

View File

@ -0,0 +1,128 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package recognizer
import (
"fmt"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
type RecognizingDecoder interface {
runtime.Decoder
// RecognizesData should return true if the input provided in the provided reader
// belongs to this decoder, or an error if the data could not be read or is ambiguous.
// Unknown is true if the data could not be determined to match the decoder type.
// Decoders should assume that they can read as much of peek as they need (as the caller
// provides) and may return unknown if the data provided is not sufficient to make a
// a determination. When peek returns EOF that may mean the end of the input or the
// end of buffered input - recognizers should return the best guess at that time.
RecognizesData(peek []byte) (ok, unknown bool, err error)
}
// NewDecoder creates a decoder that will attempt multiple decoders in an order defined
// by:
//
// 1. The decoder implements RecognizingDecoder and identifies the data
// 2. All other decoders, and any decoder that returned true for unknown.
//
// The order passed to the constructor is preserved within those priorities.
func NewDecoder(decoders ...runtime.Decoder) runtime.Decoder {
return &decoder{
decoders: decoders,
}
}
type decoder struct {
decoders []runtime.Decoder
}
var _ RecognizingDecoder = &decoder{}
func (d *decoder) RecognizesData(data []byte) (bool, bool, error) {
var (
lastErr error
anyUnknown bool
)
for _, r := range d.decoders {
switch t := r.(type) {
case RecognizingDecoder:
ok, unknown, err := t.RecognizesData(data)
if err != nil {
lastErr = err
continue
}
anyUnknown = anyUnknown || unknown
if !ok {
continue
}
return true, false, nil
}
}
return false, anyUnknown, lastErr
}
func (d *decoder) Decode(data []byte, gvk *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
var (
lastErr error
skipped []runtime.Decoder
)
// try recognizers, record any decoders we need to give a chance later
for _, r := range d.decoders {
switch t := r.(type) {
case RecognizingDecoder:
ok, unknown, err := t.RecognizesData(data)
if err != nil {
lastErr = err
continue
}
if unknown {
skipped = append(skipped, t)
continue
}
if !ok {
continue
}
return r.Decode(data, gvk, into)
default:
skipped = append(skipped, t)
}
}
// try recognizers that returned unknown or didn't recognize their data
for _, r := range skipped {
out, actual, err := r.Decode(data, gvk, into)
if err != nil {
// if we got an object back from the decoder, and the
// error was a strict decoding error (e.g. unknown or
// duplicate fields), we still consider the recognizer
// to have understood the object
if out == nil || !runtime.IsStrictDecodingError(err) {
lastErr = err
continue
}
}
return out, actual, err
}
if lastErr == nil {
lastErr = fmt.Errorf("no serialization format matched the provided data")
}
return nil, nil, lastErr
}

View File

@ -0,0 +1,136 @@
/*
Copyright 2015 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package streaming implements encoder and decoder for streams
// of runtime.Objects over io.Writer/Readers.
package streaming
import (
"bytes"
"fmt"
"io"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// Encoder is a runtime.Encoder on a stream.
type Encoder interface {
// Encode will write the provided object to the stream or return an error. It obeys the same
// contract as runtime.VersionedEncoder.
Encode(obj runtime.Object) error
}
// Decoder is a runtime.Decoder from a stream.
type Decoder interface {
// Decode will return io.EOF when no more objects are available.
Decode(defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error)
// Close closes the underlying stream.
Close() error
}
// Serializer is a factory for creating encoders and decoders that work over streams.
type Serializer interface {
NewEncoder(w io.Writer) Encoder
NewDecoder(r io.ReadCloser) Decoder
}
type decoder struct {
reader io.ReadCloser
decoder runtime.Decoder
buf []byte
maxBytes int
resetRead bool
}
// NewDecoder creates a streaming decoder that reads object chunks from r and decodes them with d.
// The reader is expected to return ErrShortRead if the provided buffer is not large enough to read
// an entire object.
func NewDecoder(r io.ReadCloser, d runtime.Decoder) Decoder {
return &decoder{
reader: r,
decoder: d,
buf: make([]byte, 1024),
maxBytes: 16 * 1024 * 1024,
}
}
var ErrObjectTooLarge = fmt.Errorf("object to decode was longer than maximum allowed size")
// Decode reads the next object from the stream and decodes it.
func (d *decoder) Decode(defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
base := 0
for {
n, err := d.reader.Read(d.buf[base:])
if err == io.ErrShortBuffer {
if n == 0 {
return nil, nil, fmt.Errorf("got short buffer with n=0, base=%d, cap=%d", base, cap(d.buf))
}
if d.resetRead {
continue
}
// double the buffer size up to maxBytes
if len(d.buf) < d.maxBytes {
base += n
d.buf = append(d.buf, make([]byte, len(d.buf))...)
continue
}
// must read the rest of the frame (until we stop getting ErrShortBuffer)
d.resetRead = true
return nil, nil, ErrObjectTooLarge
}
if err != nil {
return nil, nil, err
}
if d.resetRead {
// now that we have drained the large read, continue
d.resetRead = false
continue
}
base += n
break
}
return d.decoder.Decode(d.buf[:base], defaults, into)
}
func (d *decoder) Close() error {
return d.reader.Close()
}
type encoder struct {
writer io.Writer
encoder runtime.Encoder
buf *bytes.Buffer
}
// NewEncoder returns a new streaming encoder.
func NewEncoder(w io.Writer, e runtime.Encoder) Encoder {
return &encoder{
writer: w,
encoder: e,
buf: &bytes.Buffer{},
}
}
// Encode writes the provided object to the nested writer.
func (e *encoder) Encode(obj runtime.Object) error {
if err := e.encoder.Encode(obj, e.buf); err != nil {
return err
}
_, err := e.writer.Write(e.buf.Bytes())
e.buf.Reset()
return err
}

View File

@ -0,0 +1,290 @@
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package versioning
import (
"encoding/json"
"io"
"reflect"
"sync"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/klog/v2"
)
// NewDefaultingCodecForScheme is a convenience method for callers that are using a scheme.
func NewDefaultingCodecForScheme(
// TODO: I should be a scheme interface?
scheme *runtime.Scheme,
encoder runtime.Encoder,
decoder runtime.Decoder,
encodeVersion runtime.GroupVersioner,
decodeVersion runtime.GroupVersioner,
) runtime.Codec {
return NewCodec(encoder, decoder, runtime.UnsafeObjectConvertor(scheme), scheme, scheme, scheme, encodeVersion, decodeVersion, scheme.Name())
}
// NewCodec takes objects in their internal versions and converts them to external versions before
// serializing them. It assumes the serializer provided to it only deals with external versions.
// This class is also a serializer, but is generally used with a specific version.
func NewCodec(
encoder runtime.Encoder,
decoder runtime.Decoder,
convertor runtime.ObjectConvertor,
creater runtime.ObjectCreater,
typer runtime.ObjectTyper,
defaulter runtime.ObjectDefaulter,
encodeVersion runtime.GroupVersioner,
decodeVersion runtime.GroupVersioner,
originalSchemeName string,
) runtime.Codec {
internal := &codec{
encoder: encoder,
decoder: decoder,
convertor: convertor,
creater: creater,
typer: typer,
defaulter: defaulter,
encodeVersion: encodeVersion,
decodeVersion: decodeVersion,
identifier: identifier(encodeVersion, encoder),
originalSchemeName: originalSchemeName,
}
return internal
}
type codec struct {
encoder runtime.Encoder
decoder runtime.Decoder
convertor runtime.ObjectConvertor
creater runtime.ObjectCreater
typer runtime.ObjectTyper
defaulter runtime.ObjectDefaulter
encodeVersion runtime.GroupVersioner
decodeVersion runtime.GroupVersioner
identifier runtime.Identifier
// originalSchemeName is optional, but when filled in it holds the name of the scheme from which this codec originates
originalSchemeName string
}
var _ runtime.EncoderWithAllocator = &codec{}
var identifiersMap sync.Map
type codecIdentifier struct {
EncodeGV string `json:"encodeGV,omitempty"`
Encoder string `json:"encoder,omitempty"`
Name string `json:"name,omitempty"`
}
// identifier computes Identifier of Encoder based on codec parameters.
func identifier(encodeGV runtime.GroupVersioner, encoder runtime.Encoder) runtime.Identifier {
result := codecIdentifier{
Name: "versioning",
}
if encodeGV != nil {
result.EncodeGV = encodeGV.Identifier()
}
if encoder != nil {
result.Encoder = string(encoder.Identifier())
}
if id, ok := identifiersMap.Load(result); ok {
return id.(runtime.Identifier)
}
identifier, err := json.Marshal(result)
if err != nil {
klog.Fatalf("Failed marshaling identifier for codec: %v", err)
}
identifiersMap.Store(result, runtime.Identifier(identifier))
return runtime.Identifier(identifier)
}
// Decode attempts a decode of the object, then tries to convert it to the internal version. If into is provided and the decoding is
// successful, the returned runtime.Object will be the value passed as into. Note that this may bypass conversion if you pass an
// into that matches the serialized version.
func (c *codec) Decode(data []byte, defaultGVK *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
// If the into object is unstructured and expresses an opinion about its group/version,
// create a new instance of the type so we always exercise the conversion path (skips short-circuiting on `into == obj`)
decodeInto := into
if into != nil {
if _, ok := into.(runtime.Unstructured); ok && !into.GetObjectKind().GroupVersionKind().GroupVersion().Empty() {
decodeInto = reflect.New(reflect.TypeOf(into).Elem()).Interface().(runtime.Object)
}
}
var strictDecodingErrs []error
obj, gvk, err := c.decoder.Decode(data, defaultGVK, decodeInto)
if err != nil {
if strictErr, ok := runtime.AsStrictDecodingError(err); obj != nil && ok {
// save the strictDecodingError and let the caller decide what to do with it
strictDecodingErrs = append(strictDecodingErrs, strictErr.Errors()...)
} else {
return nil, gvk, err
}
}
if d, ok := obj.(runtime.NestedObjectDecoder); ok {
if err := d.DecodeNestedObjects(runtime.WithoutVersionDecoder{Decoder: c.decoder}); err != nil {
if strictErr, ok := runtime.AsStrictDecodingError(err); ok {
// save the strictDecodingError let and the caller decide what to do with it
strictDecodingErrs = append(strictDecodingErrs, strictErr.Errors()...)
} else {
return nil, gvk, err
}
}
}
// aggregate the strict decoding errors into one
var strictDecodingErr error
if len(strictDecodingErrs) > 0 {
strictDecodingErr = runtime.NewStrictDecodingError(strictDecodingErrs)
}
// if we specify a target, use generic conversion.
if into != nil {
// perform defaulting if requested
if c.defaulter != nil {
c.defaulter.Default(obj)
}
// Short-circuit conversion if the into object is same object
if into == obj {
return into, gvk, strictDecodingErr
}
if err := c.convertor.Convert(obj, into, c.decodeVersion); err != nil {
return nil, gvk, err
}
return into, gvk, strictDecodingErr
}
// perform defaulting if requested
if c.defaulter != nil {
c.defaulter.Default(obj)
}
out, err := c.convertor.ConvertToVersion(obj, c.decodeVersion)
if err != nil {
return nil, gvk, err
}
return out, gvk, strictDecodingErr
}
// EncodeWithAllocator ensures the provided object is output in the appropriate group and version, invoking
// conversion if necessary. Unversioned objects (according to the ObjectTyper) are output as is.
// In addition, it allows for providing a memory allocator for efficient memory usage during object serialization.
func (c *codec) EncodeWithAllocator(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
return c.encode(obj, w, memAlloc)
}
// Encode ensures the provided object is output in the appropriate group and version, invoking
// conversion if necessary. Unversioned objects (according to the ObjectTyper) are output as is.
func (c *codec) Encode(obj runtime.Object, w io.Writer) error {
return c.encode(obj, w, nil)
}
func (c *codec) encode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
if co, ok := obj.(runtime.CacheableObject); ok {
return co.CacheEncode(c.Identifier(), func(obj runtime.Object, w io.Writer) error { return c.doEncode(obj, w, memAlloc) }, w)
}
return c.doEncode(obj, w, memAlloc)
}
func (c *codec) doEncode(obj runtime.Object, w io.Writer, memAlloc runtime.MemoryAllocator) error {
encodeFn := c.encoder.Encode
if memAlloc != nil {
if encoder, supportsAllocator := c.encoder.(runtime.EncoderWithAllocator); supportsAllocator {
encodeFn = func(obj runtime.Object, w io.Writer) error {
return encoder.EncodeWithAllocator(obj, w, memAlloc)
}
} else {
klog.V(6).Infof("a memory allocator was provided but the encoder %s doesn't implement the runtime.EncoderWithAllocator, using regular encoder.Encode method", c.encoder.Identifier())
}
}
switch obj := obj.(type) {
case *runtime.Unknown:
return encodeFn(obj, w)
case runtime.Unstructured:
// An unstructured list can contain objects of multiple group version kinds. don't short-circuit just
// because the top-level type matches our desired destination type. actually send the object to the converter
// to give it a chance to convert the list items if needed.
if _, ok := obj.(*unstructured.UnstructuredList); !ok {
// avoid conversion roundtrip if GVK is the right one already or is empty (yes, this is a hack, but the old behaviour we rely on in kubectl)
objGVK := obj.GetObjectKind().GroupVersionKind()
if len(objGVK.Version) == 0 {
return encodeFn(obj, w)
}
targetGVK, ok := c.encodeVersion.KindForGroupVersionKinds([]schema.GroupVersionKind{objGVK})
if !ok {
return runtime.NewNotRegisteredGVKErrForTarget(c.originalSchemeName, objGVK, c.encodeVersion)
}
if targetGVK == objGVK {
return encodeFn(obj, w)
}
}
}
gvks, isUnversioned, err := c.typer.ObjectKinds(obj)
if err != nil {
return err
}
objectKind := obj.GetObjectKind()
old := objectKind.GroupVersionKind()
// restore the old GVK after encoding
defer objectKind.SetGroupVersionKind(old)
if c.encodeVersion == nil || isUnversioned {
if e, ok := obj.(runtime.NestedObjectEncoder); ok {
if err := e.EncodeNestedObjects(runtime.WithVersionEncoder{Encoder: c.encoder, ObjectTyper: c.typer}); err != nil {
return err
}
}
objectKind.SetGroupVersionKind(gvks[0])
return encodeFn(obj, w)
}
// Perform a conversion if necessary
out, err := c.convertor.ConvertToVersion(obj, c.encodeVersion)
if err != nil {
return err
}
if e, ok := out.(runtime.NestedObjectEncoder); ok {
if err := e.EncodeNestedObjects(runtime.WithVersionEncoder{Version: c.encodeVersion, Encoder: c.encoder, ObjectTyper: c.typer}); err != nil {
return err
}
}
// Conversion is responsible for setting the proper group, version, and kind onto the outgoing object
return encodeFn(out, w)
}
// Identifier implements runtime.Encoder interface.
func (c *codec) Identifier() runtime.Identifier {
return c.identifier
}