// Copyright 2019 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package proto import ( "errors" "fmt" "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/internal/encoding/messageset" "google.golang.org/protobuf/internal/order" "google.golang.org/protobuf/internal/pragma" "google.golang.org/protobuf/reflect/protoreflect" "google.golang.org/protobuf/runtime/protoiface" protoerrors "google.golang.org/protobuf/internal/errors" ) // MarshalOptions configures the marshaler. // // Example usage: // // b, err := MarshalOptions{Deterministic: true}.Marshal(m) type MarshalOptions struct { pragma.NoUnkeyedLiterals // AllowPartial allows messages that have missing required fields to marshal // without returning an error. If AllowPartial is false (the default), // Marshal will return an error if there are any missing required fields. AllowPartial bool // Deterministic controls whether the same message will always be // serialized to the same bytes within the same binary. // // Setting this option guarantees that repeated serialization of // the same message will return the same bytes, and that different // processes of the same binary (which may be executing on different // machines) will serialize equal messages to the same bytes. // It has no effect on the resulting size of the encoded message compared // to a non-deterministic marshal. // // Note that the deterministic serialization is NOT canonical across // languages. It is not guaranteed to remain stable over time. It is // unstable across different builds with schema changes due to unknown // fields. Users who need canonical serialization (e.g., persistent // storage in a canonical form, fingerprinting, etc.) must define // their own canonicalization specification and implement their own // serializer rather than relying on this API. // // If deterministic serialization is requested, map entries will be // sorted by keys in lexographical order. This is an implementation // detail and subject to change. Deterministic bool // UseCachedSize indicates that the result of a previous Size call // may be reused. // // Setting this option asserts that: // // 1. Size has previously been called on this message with identical // options (except for UseCachedSize itself). // // 2. The message and all its submessages have not changed in any // way since the Size call. For lazily decoded messages, accessing // a message results in decoding the message, which is a change. // // If either of these invariants is violated, // the results are undefined and may include panics or corrupted output. // // Implementations MAY take this option into account to provide // better performance, but there is no guarantee that they will do so. // There is absolutely no guarantee that Size followed by Marshal with // UseCachedSize set will perform equivalently to Marshal alone. UseCachedSize bool } // flags turns the specified MarshalOptions (user-facing) into // protoiface.MarshalInputFlags (used internally by the marshaler). // // See impl.marshalOptions.Options for the inverse operation. func (o MarshalOptions) flags() protoiface.MarshalInputFlags { var flags protoiface.MarshalInputFlags // Note: o.AllowPartial is always forced to true by MarshalOptions.marshal, // which is why it is not a part of MarshalInputFlags. if o.Deterministic { flags |= protoiface.MarshalDeterministic } if o.UseCachedSize { flags |= protoiface.MarshalUseCachedSize } return flags } // Marshal returns the wire-format encoding of m. // // This is the most common entry point for encoding a Protobuf message. // // See the [MarshalOptions] type if you need more control. func Marshal(m Message) ([]byte, error) { // Treat nil message interface as an empty message; nothing to output. if m == nil { return nil, nil } out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect()) if len(out.Buf) == 0 && err == nil { out.Buf = emptyBytesForMessage(m) } return out.Buf, err } // Marshal returns the wire-format encoding of m. func (o MarshalOptions) Marshal(m Message) ([]byte, error) { // Treat nil message interface as an empty message; nothing to output. if m == nil { return nil, nil } out, err := o.marshal(nil, m.ProtoReflect()) if len(out.Buf) == 0 && err == nil { out.Buf = emptyBytesForMessage(m) } return out.Buf, err } // emptyBytesForMessage returns a nil buffer if and only if m is invalid, // otherwise it returns a non-nil empty buffer. // // This is to assist the edge-case where user-code does the following: // // m1.OptionalBytes, _ = proto.Marshal(m2) // // where they expect the proto2 "optional_bytes" field to be populated // if any only if m2 is a valid message. func emptyBytesForMessage(m Message) []byte { if m == nil || !m.ProtoReflect().IsValid() { return nil } return emptyBuf[:] } // MarshalAppend appends the wire-format encoding of m to b, // returning the result. // // This is a less common entry point than [Marshal], which is only needed if you // need to supply your own buffers for performance reasons. func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) { // Treat nil message interface as an empty message; nothing to append. if m == nil { return b, nil } out, err := o.marshal(b, m.ProtoReflect()) return out.Buf, err } // MarshalState returns the wire-format encoding of a message. // // This method permits fine-grained control over the marshaler. // Most users should use [Marshal] instead. func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) { return o.marshal(in.Buf, in.Message) } // marshal is a centralized function that all marshal operations go through. // For profiling purposes, avoid changing the name of this function or // introducing other code paths for marshal that do not go through this. func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) { allowPartial := o.AllowPartial o.AllowPartial = true if methods := protoMethods(m); methods != nil && methods.Marshal != nil && !(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) { in := protoiface.MarshalInput{ Message: m, Buf: b, Flags: o.flags(), } if methods.Size != nil { sout := methods.Size(protoiface.SizeInput{ Message: m, Flags: in.Flags, }) if cap(b) < len(b)+sout.Size { in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size)) copy(in.Buf, b) } in.Flags |= protoiface.MarshalUseCachedSize } out, err = methods.Marshal(in) } else { out.Buf, err = o.marshalMessageSlow(b, m) } if err != nil { var mismatch *protoerrors.SizeMismatchError if errors.As(err, &mismatch) { return out, fmt.Errorf("marshaling %s: %v", string(m.Descriptor().FullName()), err) } return out, err } if allowPartial { return out, nil } return out, checkInitialized(m) } func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) { out, err := o.marshal(b, m) return out.Buf, err } // growcap scales up the capacity of a slice. // // Given a slice with a current capacity of oldcap and a desired // capacity of wantcap, growcap returns a new capacity >= wantcap. // // The algorithm is mostly identical to the one used by append as of Go 1.14. func growcap(oldcap, wantcap int) (newcap int) { if wantcap > oldcap*2 { newcap = wantcap } else if oldcap < 1024 { // The Go 1.14 runtime takes this case when len(s) < 1024, // not when cap(s) < 1024. The difference doesn't seem // significant here. newcap = oldcap * 2 } else { newcap = oldcap for 0 < newcap && newcap < wantcap { newcap += newcap / 4 } if newcap <= 0 { newcap = wantcap } } return newcap } func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) { if messageset.IsMessageSet(m.Descriptor()) { return o.marshalMessageSet(b, m) } fieldOrder := order.AnyFieldOrder if o.Deterministic { // TODO: This should use a more natural ordering like NumberFieldOrder, // but doing so breaks golden tests that make invalid assumption about // output stability of this implementation. fieldOrder = order.LegacyFieldOrder } var err error order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool { b, err = o.marshalField(b, fd, v) return err == nil }) if err != nil { return b, err } b = append(b, m.GetUnknown()...) return b, nil } func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) { switch { case fd.IsList(): return o.marshalList(b, fd, value.List()) case fd.IsMap(): return o.marshalMap(b, fd, value.Map()) default: b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()]) return o.marshalSingular(b, fd, value) } } func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) { if fd.IsPacked() && list.Len() > 0 { b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) b, pos := appendSpeculativeLength(b) for i, llen := 0, list.Len(); i < llen; i++ { var err error b, err = o.marshalSingular(b, fd, list.Get(i)) if err != nil { return b, err } } b = finishSpeculativeLength(b, pos) return b, nil } kind := fd.Kind() for i, llen := 0, list.Len(); i < llen; i++ { var err error b = protowire.AppendTag(b, fd.Number(), wireTypes[kind]) b, err = o.marshalSingular(b, fd, list.Get(i)) if err != nil { return b, err } } return b, nil } func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) { keyf := fd.MapKey() valf := fd.MapValue() keyOrder := order.AnyKeyOrder if o.Deterministic { keyOrder = order.GenericKeyOrder } var err error order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool { b = protowire.AppendTag(b, fd.Number(), protowire.BytesType) var pos int b, pos = appendSpeculativeLength(b) b, err = o.marshalField(b, keyf, key.Value()) if err != nil { return false } b, err = o.marshalField(b, valf, value) if err != nil { return false } b = finishSpeculativeLength(b, pos) return true }) return b, err } // When encoding length-prefixed fields, we speculatively set aside some number of bytes // for the length, encode the data, and then encode the length (shifting the data if necessary // to make room). const speculativeLength = 1 func appendSpeculativeLength(b []byte) ([]byte, int) { pos := len(b) b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...) return b, pos } func finishSpeculativeLength(b []byte, pos int) []byte { mlen := len(b) - pos - speculativeLength msiz := protowire.SizeVarint(uint64(mlen)) if msiz != speculativeLength { for i := 0; i < msiz-speculativeLength; i++ { b = append(b, 0) } copy(b[pos+msiz:], b[pos+speculativeLength:]) b = b[:pos+msiz+mlen] } protowire.AppendVarint(b[:pos], uint64(mlen)) return b }