build: move e2e dependencies into e2e/go.mod

Several packages are only used while running the e2e suite. These
packages are less important to update, as the they can not influence the
final executable that is part of the Ceph-CSI container-image.

By moving these dependencies out of the main Ceph-CSI go.mod, it is
easier to identify if a reported CVE affects Ceph-CSI, or only the
testing (like most of the Kubernetes CVEs).

Signed-off-by: Niels de Vos <ndevos@ibm.com>
This commit is contained in:
Niels de Vos
2025-03-04 08:57:28 +01:00
committed by mergify[bot]
parent 15da101b1b
commit bec6090996
8047 changed files with 1407827 additions and 3453 deletions

View File

@ -0,0 +1,160 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package protodelim marshals and unmarshals varint size-delimited messages.
package protodelim
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/proto"
)
// MarshalOptions is a configurable varint size-delimited marshaler.
type MarshalOptions struct{ proto.MarshalOptions }
// MarshalTo writes a varint size-delimited wire-format message to w.
// If w returns an error, MarshalTo returns it unchanged.
func (o MarshalOptions) MarshalTo(w io.Writer, m proto.Message) (int, error) {
msgBytes, err := o.MarshalOptions.Marshal(m)
if err != nil {
return 0, err
}
sizeBytes := protowire.AppendVarint(nil, uint64(len(msgBytes)))
sizeWritten, err := w.Write(sizeBytes)
if err != nil {
return sizeWritten, err
}
msgWritten, err := w.Write(msgBytes)
if err != nil {
return sizeWritten + msgWritten, err
}
return sizeWritten + msgWritten, nil
}
// MarshalTo writes a varint size-delimited wire-format message to w
// with the default options.
//
// See the documentation for [MarshalOptions.MarshalTo].
func MarshalTo(w io.Writer, m proto.Message) (int, error) {
return MarshalOptions{}.MarshalTo(w, m)
}
// UnmarshalOptions is a configurable varint size-delimited unmarshaler.
type UnmarshalOptions struct {
proto.UnmarshalOptions
// MaxSize is the maximum size in wire-format bytes of a single message.
// Unmarshaling a message larger than MaxSize will return an error.
// A zero MaxSize will default to 4 MiB.
// Setting MaxSize to -1 disables the limit.
MaxSize int64
}
const defaultMaxSize = 4 << 20 // 4 MiB, corresponds to the default gRPC max request/response size
// SizeTooLargeError is an error that is returned when the unmarshaler encounters a message size
// that is larger than its configured [UnmarshalOptions.MaxSize].
type SizeTooLargeError struct {
// Size is the varint size of the message encountered
// that was larger than the provided MaxSize.
Size uint64
// MaxSize is the MaxSize limit configured in UnmarshalOptions, which Size exceeded.
MaxSize uint64
}
func (e *SizeTooLargeError) Error() string {
return fmt.Sprintf("message size %d exceeded unmarshaler's maximum configured size %d", e.Size, e.MaxSize)
}
// Reader is the interface expected by [UnmarshalFrom].
// It is implemented by *[bufio.Reader].
type Reader interface {
io.Reader
io.ByteReader
}
// UnmarshalFrom parses and consumes a varint size-delimited wire-format message
// from r.
// The provided message must be mutable (e.g., a non-nil pointer to a message).
//
// The error is [io.EOF] error only if no bytes are read.
// If an EOF happens after reading some but not all the bytes,
// UnmarshalFrom returns a non-io.EOF error.
// In particular if r returns a non-io.EOF error, UnmarshalFrom returns it unchanged,
// and if only a size is read with no subsequent message, [io.ErrUnexpectedEOF] is returned.
func (o UnmarshalOptions) UnmarshalFrom(r Reader, m proto.Message) error {
var sizeArr [binary.MaxVarintLen64]byte
sizeBuf := sizeArr[:0]
for i := range sizeArr {
b, err := r.ReadByte()
if err != nil {
// Immediate EOF is unexpected.
if err == io.EOF && i != 0 {
break
}
return err
}
sizeBuf = append(sizeBuf, b)
if b < 0x80 {
break
}
}
size, n := protowire.ConsumeVarint(sizeBuf)
if n < 0 {
return protowire.ParseError(n)
}
maxSize := o.MaxSize
if maxSize == 0 {
maxSize = defaultMaxSize
}
if maxSize != -1 && size > uint64(maxSize) {
return errors.Wrap(&SizeTooLargeError{Size: size, MaxSize: uint64(maxSize)}, "")
}
var b []byte
var err error
if br, ok := r.(*bufio.Reader); ok {
// Use the []byte from the bufio.Reader instead of having to allocate one.
// This reduces CPU usage and allocated bytes.
b, err = br.Peek(int(size))
if err == nil {
defer br.Discard(int(size))
} else {
b = nil
}
}
if b == nil {
b = make([]byte, size)
_, err = io.ReadFull(r, b)
}
if err == io.EOF {
return io.ErrUnexpectedEOF
}
if err != nil {
return err
}
if err := o.Unmarshal(b, m); err != nil {
return err
}
return nil
}
// UnmarshalFrom parses and consumes a varint size-delimited wire-format message
// from r with the default options.
// The provided message must be mutable (e.g., a non-nil pointer to a message).
//
// See the documentation for [UnmarshalOptions.UnmarshalFrom].
func UnmarshalFrom(r Reader, m proto.Message) error {
return UnmarshalOptions{}.UnmarshalFrom(r, m)
}

View File

@ -0,0 +1,680 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protojson
import (
"encoding/base64"
"fmt"
"math"
"strconv"
"strings"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/internal/encoding/json"
"google.golang.org/protobuf/internal/encoding/messageset"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/flags"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/internal/pragma"
"google.golang.org/protobuf/internal/set"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
// Unmarshal reads the given []byte into the given [proto.Message].
// The provided message must be mutable (e.g., a non-nil pointer to a message).
func Unmarshal(b []byte, m proto.Message) error {
return UnmarshalOptions{}.Unmarshal(b, m)
}
// UnmarshalOptions is a configurable JSON format parser.
type UnmarshalOptions struct {
pragma.NoUnkeyedLiterals
// If AllowPartial is set, input for messages that will result in missing
// required fields will not return an error.
AllowPartial bool
// If DiscardUnknown is set, unknown fields and enum name values are ignored.
DiscardUnknown bool
// Resolver is used for looking up types when unmarshaling
// google.protobuf.Any messages or extension fields.
// If nil, this defaults to using protoregistry.GlobalTypes.
Resolver interface {
protoregistry.MessageTypeResolver
protoregistry.ExtensionTypeResolver
}
// RecursionLimit limits how deeply messages may be nested.
// If zero, a default limit is applied.
RecursionLimit int
}
// Unmarshal reads the given []byte and populates the given [proto.Message]
// using options in the UnmarshalOptions object.
// It will clear the message first before setting the fields.
// If it returns an error, the given message may be partially set.
// The provided message must be mutable (e.g., a non-nil pointer to a message).
func (o UnmarshalOptions) Unmarshal(b []byte, m proto.Message) error {
return o.unmarshal(b, m)
}
// unmarshal is a centralized function that all unmarshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for unmarshal that do not go through this.
func (o UnmarshalOptions) unmarshal(b []byte, m proto.Message) error {
proto.Reset(m)
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
if o.RecursionLimit == 0 {
o.RecursionLimit = protowire.DefaultRecursionLimit
}
dec := decoder{json.NewDecoder(b), o}
if err := dec.unmarshalMessage(m.ProtoReflect(), false); err != nil {
return err
}
// Check for EOF.
tok, err := dec.Read()
if err != nil {
return err
}
if tok.Kind() != json.EOF {
return dec.unexpectedTokenError(tok)
}
if o.AllowPartial {
return nil
}
return proto.CheckInitialized(m)
}
type decoder struct {
*json.Decoder
opts UnmarshalOptions
}
// newError returns an error object with position info.
func (d decoder) newError(pos int, f string, x ...any) error {
line, column := d.Position(pos)
head := fmt.Sprintf("(line %d:%d): ", line, column)
return errors.New(head+f, x...)
}
// unexpectedTokenError returns a syntax error for the given unexpected token.
func (d decoder) unexpectedTokenError(tok json.Token) error {
return d.syntaxError(tok.Pos(), "unexpected token %s", tok.RawString())
}
// syntaxError returns a syntax error for given position.
func (d decoder) syntaxError(pos int, f string, x ...any) error {
line, column := d.Position(pos)
head := fmt.Sprintf("syntax error (line %d:%d): ", line, column)
return errors.New(head+f, x...)
}
// unmarshalMessage unmarshals a message into the given protoreflect.Message.
func (d decoder) unmarshalMessage(m protoreflect.Message, skipTypeURL bool) error {
d.opts.RecursionLimit--
if d.opts.RecursionLimit < 0 {
return errors.New("exceeded max recursion depth")
}
if unmarshal := wellKnownTypeUnmarshaler(m.Descriptor().FullName()); unmarshal != nil {
return unmarshal(d, m)
}
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.ObjectOpen {
return d.unexpectedTokenError(tok)
}
messageDesc := m.Descriptor()
if !flags.ProtoLegacy && messageset.IsMessageSet(messageDesc) {
return errors.New("no support for proto1 MessageSets")
}
var seenNums set.Ints
var seenOneofs set.Ints
fieldDescs := messageDesc.Fields()
for {
// Read field name.
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
default:
return d.unexpectedTokenError(tok)
case json.ObjectClose:
return nil
case json.Name:
// Continue below.
}
name := tok.Name()
// Unmarshaling a non-custom embedded message in Any will contain the
// JSON field "@type" which should be skipped because it is not a field
// of the embedded message, but simply an artifact of the Any format.
if skipTypeURL && name == "@type" {
d.Read()
continue
}
// Get the FieldDescriptor.
var fd protoreflect.FieldDescriptor
if strings.HasPrefix(name, "[") && strings.HasSuffix(name, "]") {
// Only extension names are in [name] format.
extName := protoreflect.FullName(name[1 : len(name)-1])
extType, err := d.opts.Resolver.FindExtensionByName(extName)
if err != nil && err != protoregistry.NotFound {
return d.newError(tok.Pos(), "unable to resolve %s: %v", tok.RawString(), err)
}
if extType != nil {
fd = extType.TypeDescriptor()
if !messageDesc.ExtensionRanges().Has(fd.Number()) || fd.ContainingMessage().FullName() != messageDesc.FullName() {
return d.newError(tok.Pos(), "message %v cannot be extended by %v", messageDesc.FullName(), fd.FullName())
}
}
} else {
// The name can either be the JSON name or the proto field name.
fd = fieldDescs.ByJSONName(name)
if fd == nil {
fd = fieldDescs.ByTextName(name)
}
}
if fd == nil {
// Field is unknown.
if d.opts.DiscardUnknown {
if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
}
// Do not allow duplicate fields.
num := uint64(fd.Number())
if seenNums.Has(num) {
return d.newError(tok.Pos(), "duplicate field %v", tok.RawString())
}
seenNums.Set(num)
// No need to set values for JSON null unless the field type is
// google.protobuf.Value or google.protobuf.NullValue.
if tok, _ := d.Peek(); tok.Kind() == json.Null && !isKnownValue(fd) && !isNullValue(fd) {
d.Read()
continue
}
switch {
case fd.IsList():
list := m.Mutable(fd).List()
if err := d.unmarshalList(list, fd); err != nil {
return err
}
case fd.IsMap():
mmap := m.Mutable(fd).Map()
if err := d.unmarshalMap(mmap, fd); err != nil {
return err
}
default:
// If field is a oneof, check if it has already been set.
if od := fd.ContainingOneof(); od != nil {
idx := uint64(od.Index())
if seenOneofs.Has(idx) {
return d.newError(tok.Pos(), "error parsing %s, oneof %v is already set", tok.RawString(), od.FullName())
}
seenOneofs.Set(idx)
}
// Required or optional fields.
if err := d.unmarshalSingular(m, fd); err != nil {
return err
}
}
}
}
func isKnownValue(fd protoreflect.FieldDescriptor) bool {
md := fd.Message()
return md != nil && md.FullName() == genid.Value_message_fullname
}
func isNullValue(fd protoreflect.FieldDescriptor) bool {
ed := fd.Enum()
return ed != nil && ed.FullName() == genid.NullValue_enum_fullname
}
// unmarshalSingular unmarshals to the non-repeated field specified
// by the given FieldDescriptor.
func (d decoder) unmarshalSingular(m protoreflect.Message, fd protoreflect.FieldDescriptor) error {
var val protoreflect.Value
var err error
switch fd.Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
val = m.NewField(fd)
err = d.unmarshalMessage(val.Message(), false)
default:
val, err = d.unmarshalScalar(fd)
}
if err != nil {
return err
}
if val.IsValid() {
m.Set(fd, val)
}
return nil
}
// unmarshalScalar unmarshals to a scalar/enum protoreflect.Value specified by
// the given FieldDescriptor.
func (d decoder) unmarshalScalar(fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
const b32 int = 32
const b64 int = 64
tok, err := d.Read()
if err != nil {
return protoreflect.Value{}, err
}
kind := fd.Kind()
switch kind {
case protoreflect.BoolKind:
if tok.Kind() == json.Bool {
return protoreflect.ValueOfBool(tok.Bool()), nil
}
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
if v, ok := unmarshalInt(tok, b32); ok {
return v, nil
}
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
if v, ok := unmarshalInt(tok, b64); ok {
return v, nil
}
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
if v, ok := unmarshalUint(tok, b32); ok {
return v, nil
}
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
if v, ok := unmarshalUint(tok, b64); ok {
return v, nil
}
case protoreflect.FloatKind:
if v, ok := unmarshalFloat(tok, b32); ok {
return v, nil
}
case protoreflect.DoubleKind:
if v, ok := unmarshalFloat(tok, b64); ok {
return v, nil
}
case protoreflect.StringKind:
if tok.Kind() == json.String {
return protoreflect.ValueOfString(tok.ParsedString()), nil
}
case protoreflect.BytesKind:
if v, ok := unmarshalBytes(tok); ok {
return v, nil
}
case protoreflect.EnumKind:
if v, ok := unmarshalEnum(tok, fd, d.opts.DiscardUnknown); ok {
return v, nil
}
default:
panic(fmt.Sprintf("unmarshalScalar: invalid scalar kind %v", kind))
}
return protoreflect.Value{}, d.newError(tok.Pos(), "invalid value for %v field %v: %v", kind, fd.JSONName(), tok.RawString())
}
func unmarshalInt(tok json.Token, bitSize int) (protoreflect.Value, bool) {
switch tok.Kind() {
case json.Number:
return getInt(tok, bitSize)
case json.String:
// Decode number from string.
s := strings.TrimSpace(tok.ParsedString())
if len(s) != len(tok.ParsedString()) {
return protoreflect.Value{}, false
}
dec := json.NewDecoder([]byte(s))
tok, err := dec.Read()
if err != nil {
return protoreflect.Value{}, false
}
return getInt(tok, bitSize)
}
return protoreflect.Value{}, false
}
func getInt(tok json.Token, bitSize int) (protoreflect.Value, bool) {
n, ok := tok.Int(bitSize)
if !ok {
return protoreflect.Value{}, false
}
if bitSize == 32 {
return protoreflect.ValueOfInt32(int32(n)), true
}
return protoreflect.ValueOfInt64(n), true
}
func unmarshalUint(tok json.Token, bitSize int) (protoreflect.Value, bool) {
switch tok.Kind() {
case json.Number:
return getUint(tok, bitSize)
case json.String:
// Decode number from string.
s := strings.TrimSpace(tok.ParsedString())
if len(s) != len(tok.ParsedString()) {
return protoreflect.Value{}, false
}
dec := json.NewDecoder([]byte(s))
tok, err := dec.Read()
if err != nil {
return protoreflect.Value{}, false
}
return getUint(tok, bitSize)
}
return protoreflect.Value{}, false
}
func getUint(tok json.Token, bitSize int) (protoreflect.Value, bool) {
n, ok := tok.Uint(bitSize)
if !ok {
return protoreflect.Value{}, false
}
if bitSize == 32 {
return protoreflect.ValueOfUint32(uint32(n)), true
}
return protoreflect.ValueOfUint64(n), true
}
func unmarshalFloat(tok json.Token, bitSize int) (protoreflect.Value, bool) {
switch tok.Kind() {
case json.Number:
return getFloat(tok, bitSize)
case json.String:
s := tok.ParsedString()
switch s {
case "NaN":
if bitSize == 32 {
return protoreflect.ValueOfFloat32(float32(math.NaN())), true
}
return protoreflect.ValueOfFloat64(math.NaN()), true
case "Infinity":
if bitSize == 32 {
return protoreflect.ValueOfFloat32(float32(math.Inf(+1))), true
}
return protoreflect.ValueOfFloat64(math.Inf(+1)), true
case "-Infinity":
if bitSize == 32 {
return protoreflect.ValueOfFloat32(float32(math.Inf(-1))), true
}
return protoreflect.ValueOfFloat64(math.Inf(-1)), true
}
// Decode number from string.
if len(s) != len(strings.TrimSpace(s)) {
return protoreflect.Value{}, false
}
dec := json.NewDecoder([]byte(s))
tok, err := dec.Read()
if err != nil {
return protoreflect.Value{}, false
}
return getFloat(tok, bitSize)
}
return protoreflect.Value{}, false
}
func getFloat(tok json.Token, bitSize int) (protoreflect.Value, bool) {
n, ok := tok.Float(bitSize)
if !ok {
return protoreflect.Value{}, false
}
if bitSize == 32 {
return protoreflect.ValueOfFloat32(float32(n)), true
}
return protoreflect.ValueOfFloat64(n), true
}
func unmarshalBytes(tok json.Token) (protoreflect.Value, bool) {
if tok.Kind() != json.String {
return protoreflect.Value{}, false
}
s := tok.ParsedString()
enc := base64.StdEncoding
if strings.ContainsAny(s, "-_") {
enc = base64.URLEncoding
}
if len(s)%4 != 0 {
enc = enc.WithPadding(base64.NoPadding)
}
b, err := enc.DecodeString(s)
if err != nil {
return protoreflect.Value{}, false
}
return protoreflect.ValueOfBytes(b), true
}
func unmarshalEnum(tok json.Token, fd protoreflect.FieldDescriptor, discardUnknown bool) (protoreflect.Value, bool) {
switch tok.Kind() {
case json.String:
// Lookup EnumNumber based on name.
s := tok.ParsedString()
if enumVal := fd.Enum().Values().ByName(protoreflect.Name(s)); enumVal != nil {
return protoreflect.ValueOfEnum(enumVal.Number()), true
}
if discardUnknown {
return protoreflect.Value{}, true
}
case json.Number:
if n, ok := tok.Int(32); ok {
return protoreflect.ValueOfEnum(protoreflect.EnumNumber(n)), true
}
case json.Null:
// This is only valid for google.protobuf.NullValue.
if isNullValue(fd) {
return protoreflect.ValueOfEnum(0), true
}
}
return protoreflect.Value{}, false
}
func (d decoder) unmarshalList(list protoreflect.List, fd protoreflect.FieldDescriptor) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.ArrayOpen {
return d.unexpectedTokenError(tok)
}
switch fd.Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
for {
tok, err := d.Peek()
if err != nil {
return err
}
if tok.Kind() == json.ArrayClose {
d.Read()
return nil
}
val := list.NewElement()
if err := d.unmarshalMessage(val.Message(), false); err != nil {
return err
}
list.Append(val)
}
default:
for {
tok, err := d.Peek()
if err != nil {
return err
}
if tok.Kind() == json.ArrayClose {
d.Read()
return nil
}
val, err := d.unmarshalScalar(fd)
if err != nil {
return err
}
if val.IsValid() {
list.Append(val)
}
}
}
return nil
}
func (d decoder) unmarshalMap(mmap protoreflect.Map, fd protoreflect.FieldDescriptor) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.ObjectOpen {
return d.unexpectedTokenError(tok)
}
// Determine ahead whether map entry is a scalar type or a message type in
// order to call the appropriate unmarshalMapValue func inside the for loop
// below.
var unmarshalMapValue func() (protoreflect.Value, error)
switch fd.MapValue().Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
unmarshalMapValue = func() (protoreflect.Value, error) {
val := mmap.NewValue()
if err := d.unmarshalMessage(val.Message(), false); err != nil {
return protoreflect.Value{}, err
}
return val, nil
}
default:
unmarshalMapValue = func() (protoreflect.Value, error) {
return d.unmarshalScalar(fd.MapValue())
}
}
Loop:
for {
// Read field name.
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
default:
return d.unexpectedTokenError(tok)
case json.ObjectClose:
break Loop
case json.Name:
// Continue.
}
// Unmarshal field name.
pkey, err := d.unmarshalMapKey(tok, fd.MapKey())
if err != nil {
return err
}
// Check for duplicate field name.
if mmap.Has(pkey) {
return d.newError(tok.Pos(), "duplicate map key %v", tok.RawString())
}
// Read and unmarshal field value.
pval, err := unmarshalMapValue()
if err != nil {
return err
}
if pval.IsValid() {
mmap.Set(pkey, pval)
}
}
return nil
}
// unmarshalMapKey converts given token of Name kind into a protoreflect.MapKey.
// A map key type is any integral or string type.
func (d decoder) unmarshalMapKey(tok json.Token, fd protoreflect.FieldDescriptor) (protoreflect.MapKey, error) {
const b32 = 32
const b64 = 64
const base10 = 10
name := tok.Name()
kind := fd.Kind()
switch kind {
case protoreflect.StringKind:
return protoreflect.ValueOfString(name).MapKey(), nil
case protoreflect.BoolKind:
switch name {
case "true":
return protoreflect.ValueOfBool(true).MapKey(), nil
case "false":
return protoreflect.ValueOfBool(false).MapKey(), nil
}
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
if n, err := strconv.ParseInt(name, base10, b32); err == nil {
return protoreflect.ValueOfInt32(int32(n)).MapKey(), nil
}
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
if n, err := strconv.ParseInt(name, base10, b64); err == nil {
return protoreflect.ValueOfInt64(int64(n)).MapKey(), nil
}
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
if n, err := strconv.ParseUint(name, base10, b32); err == nil {
return protoreflect.ValueOfUint32(uint32(n)).MapKey(), nil
}
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
if n, err := strconv.ParseUint(name, base10, b64); err == nil {
return protoreflect.ValueOfUint64(uint64(n)).MapKey(), nil
}
default:
panic(fmt.Sprintf("invalid kind for map key: %v", kind))
}
return protoreflect.MapKey{}, d.newError(tok.Pos(), "invalid value for %v key: %s", kind, tok.RawString())
}

View File

@ -0,0 +1,11 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package protojson marshals and unmarshals protocol buffer messages as JSON
// format. It follows the guide at
// https://protobuf.dev/programming-guides/proto3#json.
//
// This package produces a different output than the standard [encoding/json]
// package, which does not operate correctly on protocol buffer messages.
package protojson

View File

@ -0,0 +1,380 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protojson
import (
"encoding/base64"
"fmt"
"google.golang.org/protobuf/internal/encoding/json"
"google.golang.org/protobuf/internal/encoding/messageset"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/filedesc"
"google.golang.org/protobuf/internal/flags"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/internal/order"
"google.golang.org/protobuf/internal/pragma"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const defaultIndent = " "
// Format formats the message as a multiline string.
// This function is only intended for human consumption and ignores errors.
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func Format(m proto.Message) string {
return MarshalOptions{Multiline: true}.Format(m)
}
// Marshal writes the given [proto.Message] in JSON format using default options.
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func Marshal(m proto.Message) ([]byte, error) {
return MarshalOptions{}.Marshal(m)
}
// MarshalOptions is a configurable JSON format marshaler.
type MarshalOptions struct {
pragma.NoUnkeyedLiterals
// Multiline specifies whether the marshaler should format the output in
// indented-form with every textual element on a new line.
// If Indent is an empty string, then an arbitrary indent is chosen.
Multiline bool
// Indent specifies the set of indentation characters to use in a multiline
// formatted output such that every entry is preceded by Indent and
// terminated by a newline. If non-empty, then Multiline is treated as true.
// Indent can only be composed of space or tab characters.
Indent string
// AllowPartial allows messages that have missing required fields to marshal
// without returning an error. If AllowPartial is false (the default),
// Marshal will return error if there are any missing required fields.
AllowPartial bool
// UseProtoNames uses proto field name instead of lowerCamelCase name in JSON
// field names.
UseProtoNames bool
// UseEnumNumbers emits enum values as numbers.
UseEnumNumbers bool
// EmitUnpopulated specifies whether to emit unpopulated fields. It does not
// emit unpopulated oneof fields or unpopulated extension fields.
// The JSON value emitted for unpopulated fields are as follows:
// ╔═══════╤════════════════════════════╗
// ║ JSON │ Protobuf field ║
// ╠═══════╪════════════════════════════╣
// ║ false │ proto3 boolean fields ║
// ║ 0 │ proto3 numeric fields ║
// ║ "" │ proto3 string/bytes fields ║
// ║ null │ proto2 scalar fields ║
// ║ null │ message fields ║
// ║ [] │ list fields ║
// ║ {} │ map fields ║
// ╚═══════╧════════════════════════════╝
EmitUnpopulated bool
// EmitDefaultValues specifies whether to emit default-valued primitive fields,
// empty lists, and empty maps. The fields affected are as follows:
// ╔═══════╤════════════════════════════════════════╗
// ║ JSON │ Protobuf field ║
// ╠═══════╪════════════════════════════════════════╣
// ║ false │ non-optional scalar boolean fields ║
// ║ 0 │ non-optional scalar numeric fields ║
// ║ "" │ non-optional scalar string/byte fields ║
// ║ [] │ empty repeated fields ║
// ║ {} │ empty map fields ║
// ╚═══════╧════════════════════════════════════════╝
//
// Behaves similarly to EmitUnpopulated, but does not emit "null"-value fields,
// i.e. presence-sensing fields that are omitted will remain omitted to preserve
// presence-sensing.
// EmitUnpopulated takes precedence over EmitDefaultValues since the former generates
// a strict superset of the latter.
EmitDefaultValues bool
// Resolver is used for looking up types when expanding google.protobuf.Any
// messages. If nil, this defaults to using protoregistry.GlobalTypes.
Resolver interface {
protoregistry.ExtensionTypeResolver
protoregistry.MessageTypeResolver
}
}
// Format formats the message as a string.
// This method is only intended for human consumption and ignores errors.
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func (o MarshalOptions) Format(m proto.Message) string {
if m == nil || !m.ProtoReflect().IsValid() {
return "<nil>" // invalid syntax, but okay since this is for debugging
}
o.AllowPartial = true
b, _ := o.Marshal(m)
return string(b)
}
// Marshal marshals the given [proto.Message] in the JSON format using options in
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error) {
return o.marshal(nil, m)
}
// MarshalAppend appends the JSON format encoding of m to b,
// returning the result.
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error) {
return o.marshal(b, m)
}
// marshal is a centralized function that all marshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for marshal that do not go through this.
func (o MarshalOptions) marshal(b []byte, m proto.Message) ([]byte, error) {
if o.Multiline && o.Indent == "" {
o.Indent = defaultIndent
}
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
internalEnc, err := json.NewEncoder(b, o.Indent)
if err != nil {
return nil, err
}
// Treat nil message interface as an empty message,
// in which case the output in an empty JSON object.
if m == nil {
return append(b, '{', '}'), nil
}
enc := encoder{internalEnc, o}
if err := enc.marshalMessage(m.ProtoReflect(), ""); err != nil {
return nil, err
}
if o.AllowPartial {
return enc.Bytes(), nil
}
return enc.Bytes(), proto.CheckInitialized(m)
}
type encoder struct {
*json.Encoder
opts MarshalOptions
}
// typeFieldDesc is a synthetic field descriptor used for the "@type" field.
var typeFieldDesc = func() protoreflect.FieldDescriptor {
var fd filedesc.Field
fd.L0.FullName = "@type"
fd.L0.Index = -1
fd.L1.Cardinality = protoreflect.Optional
fd.L1.Kind = protoreflect.StringKind
return &fd
}()
// typeURLFieldRanger wraps a protoreflect.Message and modifies its Range method
// to additionally iterate over a synthetic field for the type URL.
type typeURLFieldRanger struct {
order.FieldRanger
typeURL string
}
func (m typeURLFieldRanger) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {
if !f(typeFieldDesc, protoreflect.ValueOfString(m.typeURL)) {
return
}
m.FieldRanger.Range(f)
}
// unpopulatedFieldRanger wraps a protoreflect.Message and modifies its Range
// method to additionally iterate over unpopulated fields.
type unpopulatedFieldRanger struct {
protoreflect.Message
skipNull bool
}
func (m unpopulatedFieldRanger) Range(f func(protoreflect.FieldDescriptor, protoreflect.Value) bool) {
fds := m.Descriptor().Fields()
for i := 0; i < fds.Len(); i++ {
fd := fds.Get(i)
if m.Has(fd) || fd.ContainingOneof() != nil {
continue // ignore populated fields and fields within a oneofs
}
v := m.Get(fd)
if fd.HasPresence() {
if m.skipNull {
continue
}
v = protoreflect.Value{} // use invalid value to emit null
}
if !f(fd, v) {
return
}
}
m.Message.Range(f)
}
// marshalMessage marshals the fields in the given protoreflect.Message.
// If the typeURL is non-empty, then a synthetic "@type" field is injected
// containing the URL as the value.
func (e encoder) marshalMessage(m protoreflect.Message, typeURL string) error {
if !flags.ProtoLegacy && messageset.IsMessageSet(m.Descriptor()) {
return errors.New("no support for proto1 MessageSets")
}
if marshal := wellKnownTypeMarshaler(m.Descriptor().FullName()); marshal != nil {
return marshal(e, m)
}
e.StartObject()
defer e.EndObject()
var fields order.FieldRanger = m
switch {
case e.opts.EmitUnpopulated:
fields = unpopulatedFieldRanger{Message: m, skipNull: false}
case e.opts.EmitDefaultValues:
fields = unpopulatedFieldRanger{Message: m, skipNull: true}
}
if typeURL != "" {
fields = typeURLFieldRanger{fields, typeURL}
}
var err error
order.RangeFields(fields, order.IndexNameFieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
name := fd.JSONName()
if e.opts.UseProtoNames {
name = fd.TextName()
}
if err = e.WriteName(name); err != nil {
return false
}
if err = e.marshalValue(v, fd); err != nil {
return false
}
return true
})
return err
}
// marshalValue marshals the given protoreflect.Value.
func (e encoder) marshalValue(val protoreflect.Value, fd protoreflect.FieldDescriptor) error {
switch {
case fd.IsList():
return e.marshalList(val.List(), fd)
case fd.IsMap():
return e.marshalMap(val.Map(), fd)
default:
return e.marshalSingular(val, fd)
}
}
// marshalSingular marshals the given non-repeated field value. This includes
// all scalar types, enums, messages, and groups.
func (e encoder) marshalSingular(val protoreflect.Value, fd protoreflect.FieldDescriptor) error {
if !val.IsValid() {
e.WriteNull()
return nil
}
switch kind := fd.Kind(); kind {
case protoreflect.BoolKind:
e.WriteBool(val.Bool())
case protoreflect.StringKind:
if e.WriteString(val.String()) != nil {
return errors.InvalidUTF8(string(fd.FullName()))
}
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
e.WriteInt(val.Int())
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
e.WriteUint(val.Uint())
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Uint64Kind,
protoreflect.Sfixed64Kind, protoreflect.Fixed64Kind:
// 64-bit integers are written out as JSON string.
e.WriteString(val.String())
case protoreflect.FloatKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
e.WriteFloat(val.Float(), 32)
case protoreflect.DoubleKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
e.WriteFloat(val.Float(), 64)
case protoreflect.BytesKind:
e.WriteString(base64.StdEncoding.EncodeToString(val.Bytes()))
case protoreflect.EnumKind:
if fd.Enum().FullName() == genid.NullValue_enum_fullname {
e.WriteNull()
} else {
desc := fd.Enum().Values().ByNumber(val.Enum())
if e.opts.UseEnumNumbers || desc == nil {
e.WriteInt(int64(val.Enum()))
} else {
e.WriteString(string(desc.Name()))
}
}
case protoreflect.MessageKind, protoreflect.GroupKind:
if err := e.marshalMessage(val.Message(), ""); err != nil {
return err
}
default:
panic(fmt.Sprintf("%v has unknown kind: %v", fd.FullName(), kind))
}
return nil
}
// marshalList marshals the given protoreflect.List.
func (e encoder) marshalList(list protoreflect.List, fd protoreflect.FieldDescriptor) error {
e.StartArray()
defer e.EndArray()
for i := 0; i < list.Len(); i++ {
item := list.Get(i)
if err := e.marshalSingular(item, fd); err != nil {
return err
}
}
return nil
}
// marshalMap marshals given protoreflect.Map.
func (e encoder) marshalMap(mmap protoreflect.Map, fd protoreflect.FieldDescriptor) error {
e.StartObject()
defer e.EndObject()
var err error
order.RangeEntries(mmap, order.GenericKeyOrder, func(k protoreflect.MapKey, v protoreflect.Value) bool {
if err = e.WriteName(k.String()); err != nil {
return false
}
if err = e.marshalSingular(v, fd.MapValue()); err != nil {
return false
}
return true
})
return err
}

View File

@ -0,0 +1,880 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package protojson
import (
"bytes"
"fmt"
"math"
"strconv"
"strings"
"time"
"google.golang.org/protobuf/internal/encoding/json"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/internal/strs"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
)
type marshalFunc func(encoder, protoreflect.Message) error
// wellKnownTypeMarshaler returns a marshal function if the message type
// has specialized serialization behavior. It returns nil otherwise.
func wellKnownTypeMarshaler(name protoreflect.FullName) marshalFunc {
if name.Parent() == genid.GoogleProtobuf_package {
switch name.Name() {
case genid.Any_message_name:
return encoder.marshalAny
case genid.Timestamp_message_name:
return encoder.marshalTimestamp
case genid.Duration_message_name:
return encoder.marshalDuration
case genid.BoolValue_message_name,
genid.Int32Value_message_name,
genid.Int64Value_message_name,
genid.UInt32Value_message_name,
genid.UInt64Value_message_name,
genid.FloatValue_message_name,
genid.DoubleValue_message_name,
genid.StringValue_message_name,
genid.BytesValue_message_name:
return encoder.marshalWrapperType
case genid.Struct_message_name:
return encoder.marshalStruct
case genid.ListValue_message_name:
return encoder.marshalListValue
case genid.Value_message_name:
return encoder.marshalKnownValue
case genid.FieldMask_message_name:
return encoder.marshalFieldMask
case genid.Empty_message_name:
return encoder.marshalEmpty
}
}
return nil
}
type unmarshalFunc func(decoder, protoreflect.Message) error
// wellKnownTypeUnmarshaler returns a unmarshal function if the message type
// has specialized serialization behavior. It returns nil otherwise.
func wellKnownTypeUnmarshaler(name protoreflect.FullName) unmarshalFunc {
if name.Parent() == genid.GoogleProtobuf_package {
switch name.Name() {
case genid.Any_message_name:
return decoder.unmarshalAny
case genid.Timestamp_message_name:
return decoder.unmarshalTimestamp
case genid.Duration_message_name:
return decoder.unmarshalDuration
case genid.BoolValue_message_name,
genid.Int32Value_message_name,
genid.Int64Value_message_name,
genid.UInt32Value_message_name,
genid.UInt64Value_message_name,
genid.FloatValue_message_name,
genid.DoubleValue_message_name,
genid.StringValue_message_name,
genid.BytesValue_message_name:
return decoder.unmarshalWrapperType
case genid.Struct_message_name:
return decoder.unmarshalStruct
case genid.ListValue_message_name:
return decoder.unmarshalListValue
case genid.Value_message_name:
return decoder.unmarshalKnownValue
case genid.FieldMask_message_name:
return decoder.unmarshalFieldMask
case genid.Empty_message_name:
return decoder.unmarshalEmpty
}
}
return nil
}
// The JSON representation of an Any message uses the regular representation of
// the deserialized, embedded message, with an additional field `@type` which
// contains the type URL. If the embedded message type is well-known and has a
// custom JSON representation, that representation will be embedded adding a
// field `value` which holds the custom JSON in addition to the `@type` field.
func (e encoder) marshalAny(m protoreflect.Message) error {
fds := m.Descriptor().Fields()
fdType := fds.ByNumber(genid.Any_TypeUrl_field_number)
fdValue := fds.ByNumber(genid.Any_Value_field_number)
if !m.Has(fdType) {
if !m.Has(fdValue) {
// If message is empty, marshal out empty JSON object.
e.StartObject()
e.EndObject()
return nil
} else {
// Return error if type_url field is not set, but value is set.
return errors.New("%s: %v is not set", genid.Any_message_fullname, genid.Any_TypeUrl_field_name)
}
}
typeVal := m.Get(fdType)
valueVal := m.Get(fdValue)
// Resolve the type in order to unmarshal value field.
typeURL := typeVal.String()
emt, err := e.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
return errors.New("%s: unable to resolve %q: %v", genid.Any_message_fullname, typeURL, err)
}
em := emt.New()
err = proto.UnmarshalOptions{
AllowPartial: true, // never check required fields inside an Any
Resolver: e.opts.Resolver,
}.Unmarshal(valueVal.Bytes(), em.Interface())
if err != nil {
return errors.New("%s: unable to unmarshal %q: %v", genid.Any_message_fullname, typeURL, err)
}
// If type of value has custom JSON encoding, marshal out a field "value"
// with corresponding custom JSON encoding of the embedded message as a
// field.
if marshal := wellKnownTypeMarshaler(emt.Descriptor().FullName()); marshal != nil {
e.StartObject()
defer e.EndObject()
// Marshal out @type field.
e.WriteName("@type")
if err := e.WriteString(typeURL); err != nil {
return err
}
e.WriteName("value")
return marshal(e, em)
}
// Else, marshal out the embedded message's fields in this Any object.
if err := e.marshalMessage(em, typeURL); err != nil {
return err
}
return nil
}
func (d decoder) unmarshalAny(m protoreflect.Message) error {
// Peek to check for json.ObjectOpen to avoid advancing a read.
start, err := d.Peek()
if err != nil {
return err
}
if start.Kind() != json.ObjectOpen {
return d.unexpectedTokenError(start)
}
// Use another decoder to parse the unread bytes for @type field. This
// avoids advancing a read from current decoder because the current JSON
// object may contain the fields of the embedded type.
dec := decoder{d.Clone(), UnmarshalOptions{RecursionLimit: d.opts.RecursionLimit}}
tok, err := findTypeURL(dec)
switch err {
case errEmptyObject:
// An empty JSON object translates to an empty Any message.
d.Read() // Read json.ObjectOpen.
d.Read() // Read json.ObjectClose.
return nil
case errMissingType:
if d.opts.DiscardUnknown {
// Treat all fields as unknowns, similar to an empty object.
return d.skipJSONValue()
}
// Use start.Pos() for line position.
return d.newError(start.Pos(), err.Error())
default:
if err != nil {
return err
}
}
typeURL := tok.ParsedString()
emt, err := d.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
return d.newError(tok.Pos(), "unable to resolve %v: %q", tok.RawString(), err)
}
// Create new message for the embedded message type and unmarshal into it.
em := emt.New()
if unmarshal := wellKnownTypeUnmarshaler(emt.Descriptor().FullName()); unmarshal != nil {
// If embedded message is a custom type,
// unmarshal the JSON "value" field into it.
if err := d.unmarshalAnyValue(unmarshal, em); err != nil {
return err
}
} else {
// Else unmarshal the current JSON object into it.
if err := d.unmarshalMessage(em, true); err != nil {
return err
}
}
// Serialize the embedded message and assign the resulting bytes to the
// proto value field.
b, err := proto.MarshalOptions{
AllowPartial: true, // No need to check required fields inside an Any.
Deterministic: true,
}.Marshal(em.Interface())
if err != nil {
return d.newError(start.Pos(), "error in marshaling Any.value field: %v", err)
}
fds := m.Descriptor().Fields()
fdType := fds.ByNumber(genid.Any_TypeUrl_field_number)
fdValue := fds.ByNumber(genid.Any_Value_field_number)
m.Set(fdType, protoreflect.ValueOfString(typeURL))
m.Set(fdValue, protoreflect.ValueOfBytes(b))
return nil
}
var errEmptyObject = fmt.Errorf(`empty object`)
var errMissingType = fmt.Errorf(`missing "@type" field`)
// findTypeURL returns the token for the "@type" field value from the given
// JSON bytes. It is expected that the given bytes start with json.ObjectOpen.
// It returns errEmptyObject if the JSON object is empty or errMissingType if
// @type field does not exist. It returns other error if the @type field is not
// valid or other decoding issues.
func findTypeURL(d decoder) (json.Token, error) {
var typeURL string
var typeTok json.Token
numFields := 0
// Skip start object.
d.Read()
Loop:
for {
tok, err := d.Read()
if err != nil {
return json.Token{}, err
}
switch tok.Kind() {
case json.ObjectClose:
if typeURL == "" {
// Did not find @type field.
if numFields > 0 {
return json.Token{}, errMissingType
}
return json.Token{}, errEmptyObject
}
break Loop
case json.Name:
numFields++
if tok.Name() != "@type" {
// Skip value.
if err := d.skipJSONValue(); err != nil {
return json.Token{}, err
}
continue
}
// Return error if this was previously set already.
if typeURL != "" {
return json.Token{}, d.newError(tok.Pos(), `duplicate "@type" field`)
}
// Read field value.
tok, err := d.Read()
if err != nil {
return json.Token{}, err
}
if tok.Kind() != json.String {
return json.Token{}, d.newError(tok.Pos(), `@type field value is not a string: %v`, tok.RawString())
}
typeURL = tok.ParsedString()
if typeURL == "" {
return json.Token{}, d.newError(tok.Pos(), `@type field contains empty value`)
}
typeTok = tok
}
}
return typeTok, nil
}
// skipJSONValue parses a JSON value (null, boolean, string, number, object and
// array) in order to advance the read to the next JSON value. It relies on
// the decoder returning an error if the types are not in valid sequence.
func (d decoder) skipJSONValue() error {
var open int
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case json.ObjectClose, json.ArrayClose:
open--
case json.ObjectOpen, json.ArrayOpen:
open++
if open > d.opts.RecursionLimit {
return errors.New("exceeded max recursion depth")
}
case json.EOF:
// This can only happen if there's a bug in Decoder.Read.
// Avoid an infinite loop if this does happen.
return errors.New("unexpected EOF")
}
if open == 0 {
return nil
}
}
}
// unmarshalAnyValue unmarshals the given custom-type message from the JSON
// object's "value" field.
func (d decoder) unmarshalAnyValue(unmarshal unmarshalFunc, m protoreflect.Message) error {
// Skip ObjectOpen, and start reading the fields.
d.Read()
var found bool // Used for detecting duplicate "value".
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case json.ObjectClose:
if !found {
// We tolerate an omitted `value` field with the google.protobuf.Empty Well-Known-Type,
// for compatibility with other proto runtimes that have interpreted the spec differently.
if m.Descriptor().FullName() != genid.Empty_message_fullname {
return d.newError(tok.Pos(), `missing "value" field`)
}
}
return nil
case json.Name:
switch tok.Name() {
case "@type":
// Skip the value as this was previously parsed already.
d.Read()
case "value":
if found {
return d.newError(tok.Pos(), `duplicate "value" field`)
}
// Unmarshal the field value into the given message.
if err := unmarshal(d, m); err != nil {
return err
}
found = true
default:
if d.opts.DiscardUnknown {
if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
}
}
}
}
// Wrapper types are encoded as JSON primitives like string, number or boolean.
func (e encoder) marshalWrapperType(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.WrapperValue_Value_field_number)
val := m.Get(fd)
return e.marshalSingular(val, fd)
}
func (d decoder) unmarshalWrapperType(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.WrapperValue_Value_field_number)
val, err := d.unmarshalScalar(fd)
if err != nil {
return err
}
m.Set(fd, val)
return nil
}
// The JSON representation for Empty is an empty JSON object.
func (e encoder) marshalEmpty(protoreflect.Message) error {
e.StartObject()
e.EndObject()
return nil
}
func (d decoder) unmarshalEmpty(protoreflect.Message) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.ObjectOpen {
return d.unexpectedTokenError(tok)
}
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case json.ObjectClose:
return nil
case json.Name:
if d.opts.DiscardUnknown {
if err := d.skipJSONValue(); err != nil {
return err
}
continue
}
return d.newError(tok.Pos(), "unknown field %v", tok.RawString())
default:
return d.unexpectedTokenError(tok)
}
}
}
// The JSON representation for Struct is a JSON object that contains the encoded
// Struct.fields map and follows the serialization rules for a map.
func (e encoder) marshalStruct(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.Struct_Fields_field_number)
return e.marshalMap(m.Get(fd).Map(), fd)
}
func (d decoder) unmarshalStruct(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.Struct_Fields_field_number)
return d.unmarshalMap(m.Mutable(fd).Map(), fd)
}
// The JSON representation for ListValue is JSON array that contains the encoded
// ListValue.values repeated field and follows the serialization rules for a
// repeated field.
func (e encoder) marshalListValue(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.ListValue_Values_field_number)
return e.marshalList(m.Get(fd).List(), fd)
}
func (d decoder) unmarshalListValue(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.ListValue_Values_field_number)
return d.unmarshalList(m.Mutable(fd).List(), fd)
}
// The JSON representation for a Value is dependent on the oneof field that is
// set. Each of the field in the oneof has its own custom serialization rule. A
// Value message needs to be a oneof field set, else it is an error.
func (e encoder) marshalKnownValue(m protoreflect.Message) error {
od := m.Descriptor().Oneofs().ByName(genid.Value_Kind_oneof_name)
fd := m.WhichOneof(od)
if fd == nil {
return errors.New("%s: none of the oneof fields is set", genid.Value_message_fullname)
}
if fd.Number() == genid.Value_NumberValue_field_number {
if v := m.Get(fd).Float(); math.IsNaN(v) || math.IsInf(v, 0) {
return errors.New("%s: invalid %v value", genid.Value_NumberValue_field_fullname, v)
}
}
return e.marshalSingular(m.Get(fd), fd)
}
func (d decoder) unmarshalKnownValue(m protoreflect.Message) error {
tok, err := d.Peek()
if err != nil {
return err
}
var fd protoreflect.FieldDescriptor
var val protoreflect.Value
switch tok.Kind() {
case json.Null:
d.Read()
fd = m.Descriptor().Fields().ByNumber(genid.Value_NullValue_field_number)
val = protoreflect.ValueOfEnum(0)
case json.Bool:
tok, err := d.Read()
if err != nil {
return err
}
fd = m.Descriptor().Fields().ByNumber(genid.Value_BoolValue_field_number)
val = protoreflect.ValueOfBool(tok.Bool())
case json.Number:
tok, err := d.Read()
if err != nil {
return err
}
fd = m.Descriptor().Fields().ByNumber(genid.Value_NumberValue_field_number)
var ok bool
val, ok = unmarshalFloat(tok, 64)
if !ok {
return d.newError(tok.Pos(), "invalid %v: %v", genid.Value_message_fullname, tok.RawString())
}
case json.String:
// A JSON string may have been encoded from the number_value field,
// e.g. "NaN", "Infinity", etc. Parsing a proto double type also allows
// for it to be in JSON string form. Given this custom encoding spec,
// however, there is no way to identify that and hence a JSON string is
// always assigned to the string_value field, which means that certain
// encoding cannot be parsed back to the same field.
tok, err := d.Read()
if err != nil {
return err
}
fd = m.Descriptor().Fields().ByNumber(genid.Value_StringValue_field_number)
val = protoreflect.ValueOfString(tok.ParsedString())
case json.ObjectOpen:
fd = m.Descriptor().Fields().ByNumber(genid.Value_StructValue_field_number)
val = m.NewField(fd)
if err := d.unmarshalStruct(val.Message()); err != nil {
return err
}
case json.ArrayOpen:
fd = m.Descriptor().Fields().ByNumber(genid.Value_ListValue_field_number)
val = m.NewField(fd)
if err := d.unmarshalListValue(val.Message()); err != nil {
return err
}
default:
return d.newError(tok.Pos(), "invalid %v: %v", genid.Value_message_fullname, tok.RawString())
}
m.Set(fd, val)
return nil
}
// The JSON representation for a Duration is a JSON string that ends in the
// suffix "s" (indicating seconds) and is preceded by the number of seconds,
// with nanoseconds expressed as fractional seconds.
//
// Durations less than one second are represented with a 0 seconds field and a
// positive or negative nanos field. For durations of one second or more, a
// non-zero value for the nanos field must be of the same sign as the seconds
// field.
//
// Duration.seconds must be from -315,576,000,000 to +315,576,000,000 inclusive.
// Duration.nanos must be from -999,999,999 to +999,999,999 inclusive.
const (
secondsInNanos = 999999999
maxSecondsInDuration = 315576000000
)
func (e encoder) marshalDuration(m protoreflect.Message) error {
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(genid.Duration_Seconds_field_number)
fdNanos := fds.ByNumber(genid.Duration_Nanos_field_number)
secsVal := m.Get(fdSeconds)
nanosVal := m.Get(fdNanos)
secs := secsVal.Int()
nanos := nanosVal.Int()
if secs < -maxSecondsInDuration || secs > maxSecondsInDuration {
return errors.New("%s: seconds out of range %v", genid.Duration_message_fullname, secs)
}
if nanos < -secondsInNanos || nanos > secondsInNanos {
return errors.New("%s: nanos out of range %v", genid.Duration_message_fullname, nanos)
}
if (secs > 0 && nanos < 0) || (secs < 0 && nanos > 0) {
return errors.New("%s: signs of seconds and nanos do not match", genid.Duration_message_fullname)
}
// Generated output always contains 0, 3, 6, or 9 fractional digits,
// depending on required precision, followed by the suffix "s".
var sign string
if secs < 0 || nanos < 0 {
sign, secs, nanos = "-", -1*secs, -1*nanos
}
x := fmt.Sprintf("%s%d.%09d", sign, secs, nanos)
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
e.WriteString(x + "s")
return nil
}
func (d decoder) unmarshalDuration(m protoreflect.Message) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.String {
return d.unexpectedTokenError(tok)
}
secs, nanos, ok := parseDuration(tok.ParsedString())
if !ok {
return d.newError(tok.Pos(), "invalid %v value %v", genid.Duration_message_fullname, tok.RawString())
}
// Validate seconds. No need to validate nanos because parseDuration would
// have covered that already.
if secs < -maxSecondsInDuration || secs > maxSecondsInDuration {
return d.newError(tok.Pos(), "%v value out of range: %v", genid.Duration_message_fullname, tok.RawString())
}
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(genid.Duration_Seconds_field_number)
fdNanos := fds.ByNumber(genid.Duration_Nanos_field_number)
m.Set(fdSeconds, protoreflect.ValueOfInt64(secs))
m.Set(fdNanos, protoreflect.ValueOfInt32(nanos))
return nil
}
// parseDuration parses the given input string for seconds and nanoseconds value
// for the Duration JSON format. The format is a decimal number with a suffix
// 's'. It can have optional plus/minus sign. There needs to be at least an
// integer or fractional part. Fractional part is limited to 9 digits only for
// nanoseconds precision, regardless of whether there are trailing zero digits.
// Example values are 1s, 0.1s, 1.s, .1s, +1s, -1s, -.1s.
func parseDuration(input string) (int64, int32, bool) {
b := []byte(input)
size := len(b)
if size < 2 {
return 0, 0, false
}
if b[size-1] != 's' {
return 0, 0, false
}
b = b[:size-1]
// Read optional plus/minus symbol.
var neg bool
switch b[0] {
case '-':
neg = true
b = b[1:]
case '+':
b = b[1:]
}
if len(b) == 0 {
return 0, 0, false
}
// Read the integer part.
var intp []byte
switch {
case b[0] == '0':
b = b[1:]
case '1' <= b[0] && b[0] <= '9':
intp = b[0:]
b = b[1:]
n := 1
for len(b) > 0 && '0' <= b[0] && b[0] <= '9' {
n++
b = b[1:]
}
intp = intp[:n]
case b[0] == '.':
// Continue below.
default:
return 0, 0, false
}
hasFrac := false
var frac [9]byte
if len(b) > 0 {
if b[0] != '.' {
return 0, 0, false
}
// Read the fractional part.
b = b[1:]
n := 0
for len(b) > 0 && n < 9 && '0' <= b[0] && b[0] <= '9' {
frac[n] = b[0]
n++
b = b[1:]
}
// It is not valid if there are more bytes left.
if len(b) > 0 {
return 0, 0, false
}
// Pad fractional part with 0s.
for i := n; i < 9; i++ {
frac[i] = '0'
}
hasFrac = true
}
var secs int64
if len(intp) > 0 {
var err error
secs, err = strconv.ParseInt(string(intp), 10, 64)
if err != nil {
return 0, 0, false
}
}
var nanos int64
if hasFrac {
nanob := bytes.TrimLeft(frac[:], "0")
if len(nanob) > 0 {
var err error
nanos, err = strconv.ParseInt(string(nanob), 10, 32)
if err != nil {
return 0, 0, false
}
}
}
if neg {
if secs > 0 {
secs = -secs
}
if nanos > 0 {
nanos = -nanos
}
}
return secs, int32(nanos), true
}
// The JSON representation for a Timestamp is a JSON string in the RFC 3339
// format, i.e. "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" where
// {year} is always expressed using four digits while {month}, {day}, {hour},
// {min}, and {sec} are zero-padded to two digits each. The fractional seconds,
// which can go up to 9 digits, up to 1 nanosecond resolution, is optional. The
// "Z" suffix indicates the timezone ("UTC"); the timezone is required. Encoding
// should always use UTC (as indicated by "Z") and a decoder should be able to
// accept both UTC and other timezones (as indicated by an offset).
//
// Timestamp.seconds must be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z
// inclusive.
// Timestamp.nanos must be from 0 to 999,999,999 inclusive.
const (
maxTimestampSeconds = 253402300799
minTimestampSeconds = -62135596800
)
func (e encoder) marshalTimestamp(m protoreflect.Message) error {
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(genid.Timestamp_Seconds_field_number)
fdNanos := fds.ByNumber(genid.Timestamp_Nanos_field_number)
secsVal := m.Get(fdSeconds)
nanosVal := m.Get(fdNanos)
secs := secsVal.Int()
nanos := nanosVal.Int()
if secs < minTimestampSeconds || secs > maxTimestampSeconds {
return errors.New("%s: seconds out of range %v", genid.Timestamp_message_fullname, secs)
}
if nanos < 0 || nanos > secondsInNanos {
return errors.New("%s: nanos out of range %v", genid.Timestamp_message_fullname, nanos)
}
// Uses RFC 3339, where generated output will be Z-normalized and uses 0, 3,
// 6 or 9 fractional digits.
t := time.Unix(secs, nanos).UTC()
x := t.Format("2006-01-02T15:04:05.000000000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, "000")
x = strings.TrimSuffix(x, ".000")
e.WriteString(x + "Z")
return nil
}
func (d decoder) unmarshalTimestamp(m protoreflect.Message) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.String {
return d.unexpectedTokenError(tok)
}
s := tok.ParsedString()
t, err := time.Parse(time.RFC3339Nano, s)
if err != nil {
return d.newError(tok.Pos(), "invalid %v value %v", genid.Timestamp_message_fullname, tok.RawString())
}
// Validate seconds.
secs := t.Unix()
if secs < minTimestampSeconds || secs > maxTimestampSeconds {
return d.newError(tok.Pos(), "%v value out of range: %v", genid.Timestamp_message_fullname, tok.RawString())
}
// Validate subseconds.
i := strings.LastIndexByte(s, '.') // start of subsecond field
j := strings.LastIndexAny(s, "Z-+") // start of timezone field
if i >= 0 && j >= i && j-i > len(".999999999") {
return d.newError(tok.Pos(), "invalid %v value %v", genid.Timestamp_message_fullname, tok.RawString())
}
fds := m.Descriptor().Fields()
fdSeconds := fds.ByNumber(genid.Timestamp_Seconds_field_number)
fdNanos := fds.ByNumber(genid.Timestamp_Nanos_field_number)
m.Set(fdSeconds, protoreflect.ValueOfInt64(secs))
m.Set(fdNanos, protoreflect.ValueOfInt32(int32(t.Nanosecond())))
return nil
}
// The JSON representation for a FieldMask is a JSON string where paths are
// separated by a comma. Fields name in each path are converted to/from
// lower-camel naming conventions. Encoding should fail if the path name would
// end up differently after a round-trip.
func (e encoder) marshalFieldMask(m protoreflect.Message) error {
fd := m.Descriptor().Fields().ByNumber(genid.FieldMask_Paths_field_number)
list := m.Get(fd).List()
paths := make([]string, 0, list.Len())
for i := 0; i < list.Len(); i++ {
s := list.Get(i).String()
if !protoreflect.FullName(s).IsValid() {
return errors.New("%s contains invalid path: %q", genid.FieldMask_Paths_field_fullname, s)
}
// Return error if conversion to camelCase is not reversible.
cc := strs.JSONCamelCase(s)
if s != strs.JSONSnakeCase(cc) {
return errors.New("%s contains irreversible value %q", genid.FieldMask_Paths_field_fullname, s)
}
paths = append(paths, cc)
}
e.WriteString(strings.Join(paths, ","))
return nil
}
func (d decoder) unmarshalFieldMask(m protoreflect.Message) error {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != json.String {
return d.unexpectedTokenError(tok)
}
str := strings.TrimSpace(tok.ParsedString())
if str == "" {
return nil
}
paths := strings.Split(str, ",")
fd := m.Descriptor().Fields().ByNumber(genid.FieldMask_Paths_field_number)
list := m.Mutable(fd).List()
for _, s0 := range paths {
s := strs.JSONSnakeCase(s0)
if strings.Contains(s0, "_") || !protoreflect.FullName(s).IsValid() {
return d.newError(tok.Pos(), "%v contains invalid path: %q", genid.FieldMask_Paths_field_fullname, s0)
}
list.Append(protoreflect.ValueOfString(s))
}
return nil
}

View File

@ -0,0 +1,767 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package prototext
import (
"fmt"
"unicode/utf8"
"google.golang.org/protobuf/internal/encoding/messageset"
"google.golang.org/protobuf/internal/encoding/text"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/flags"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/internal/pragma"
"google.golang.org/protobuf/internal/set"
"google.golang.org/protobuf/internal/strs"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
// Unmarshal reads the given []byte into the given [proto.Message].
// The provided message must be mutable (e.g., a non-nil pointer to a message).
func Unmarshal(b []byte, m proto.Message) error {
return UnmarshalOptions{}.Unmarshal(b, m)
}
// UnmarshalOptions is a configurable textproto format unmarshaler.
type UnmarshalOptions struct {
pragma.NoUnkeyedLiterals
// AllowPartial accepts input for messages that will result in missing
// required fields. If AllowPartial is false (the default), Unmarshal will
// return error if there are any missing required fields.
AllowPartial bool
// DiscardUnknown specifies whether to ignore unknown fields when parsing.
// An unknown field is any field whose field name or field number does not
// resolve to any known or extension field in the message.
// By default, unmarshal rejects unknown fields as an error.
DiscardUnknown bool
// Resolver is used for looking up types when unmarshaling
// google.protobuf.Any messages or extension fields.
// If nil, this defaults to using protoregistry.GlobalTypes.
Resolver interface {
protoregistry.MessageTypeResolver
protoregistry.ExtensionTypeResolver
}
}
// Unmarshal reads the given []byte and populates the given [proto.Message]
// using options in the UnmarshalOptions object.
// The provided message must be mutable (e.g., a non-nil pointer to a message).
func (o UnmarshalOptions) Unmarshal(b []byte, m proto.Message) error {
return o.unmarshal(b, m)
}
// unmarshal is a centralized function that all unmarshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for unmarshal that do not go through this.
func (o UnmarshalOptions) unmarshal(b []byte, m proto.Message) error {
proto.Reset(m)
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
dec := decoder{text.NewDecoder(b), o}
if err := dec.unmarshalMessage(m.ProtoReflect(), false); err != nil {
return err
}
if o.AllowPartial {
return nil
}
return proto.CheckInitialized(m)
}
type decoder struct {
*text.Decoder
opts UnmarshalOptions
}
// newError returns an error object with position info.
func (d decoder) newError(pos int, f string, x ...any) error {
line, column := d.Position(pos)
head := fmt.Sprintf("(line %d:%d): ", line, column)
return errors.New(head+f, x...)
}
// unexpectedTokenError returns a syntax error for the given unexpected token.
func (d decoder) unexpectedTokenError(tok text.Token) error {
return d.syntaxError(tok.Pos(), "unexpected token: %s", tok.RawString())
}
// syntaxError returns a syntax error for given position.
func (d decoder) syntaxError(pos int, f string, x ...any) error {
line, column := d.Position(pos)
head := fmt.Sprintf("syntax error (line %d:%d): ", line, column)
return errors.New(head+f, x...)
}
// unmarshalMessage unmarshals into the given protoreflect.Message.
func (d decoder) unmarshalMessage(m protoreflect.Message, checkDelims bool) error {
messageDesc := m.Descriptor()
if !flags.ProtoLegacy && messageset.IsMessageSet(messageDesc) {
return errors.New("no support for proto1 MessageSets")
}
if messageDesc.FullName() == genid.Any_message_fullname {
return d.unmarshalAny(m, checkDelims)
}
if checkDelims {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != text.MessageOpen {
return d.unexpectedTokenError(tok)
}
}
var seenNums set.Ints
var seenOneofs set.Ints
fieldDescs := messageDesc.Fields()
for {
// Read field name.
tok, err := d.Read()
if err != nil {
return err
}
switch typ := tok.Kind(); typ {
case text.Name:
// Continue below.
case text.EOF:
if checkDelims {
return text.ErrUnexpectedEOF
}
return nil
default:
if checkDelims && typ == text.MessageClose {
return nil
}
return d.unexpectedTokenError(tok)
}
// Resolve the field descriptor.
var name protoreflect.Name
var fd protoreflect.FieldDescriptor
var xt protoreflect.ExtensionType
var xtErr error
var isFieldNumberName bool
switch tok.NameKind() {
case text.IdentName:
name = protoreflect.Name(tok.IdentName())
fd = fieldDescs.ByTextName(string(name))
case text.TypeName:
// Handle extensions only. This code path is not for Any.
xt, xtErr = d.opts.Resolver.FindExtensionByName(protoreflect.FullName(tok.TypeName()))
case text.FieldNumber:
isFieldNumberName = true
num := protoreflect.FieldNumber(tok.FieldNumber())
if !num.IsValid() {
return d.newError(tok.Pos(), "invalid field number: %d", num)
}
fd = fieldDescs.ByNumber(num)
if fd == nil {
xt, xtErr = d.opts.Resolver.FindExtensionByNumber(messageDesc.FullName(), num)
}
}
if xt != nil {
fd = xt.TypeDescriptor()
if !messageDesc.ExtensionRanges().Has(fd.Number()) || fd.ContainingMessage().FullName() != messageDesc.FullName() {
return d.newError(tok.Pos(), "message %v cannot be extended by %v", messageDesc.FullName(), fd.FullName())
}
} else if xtErr != nil && xtErr != protoregistry.NotFound {
return d.newError(tok.Pos(), "unable to resolve [%s]: %v", tok.RawString(), xtErr)
}
// Handle unknown fields.
if fd == nil {
if d.opts.DiscardUnknown || messageDesc.ReservedNames().Has(name) {
d.skipValue()
continue
}
return d.newError(tok.Pos(), "unknown field: %v", tok.RawString())
}
// Handle fields identified by field number.
if isFieldNumberName {
// TODO: Add an option to permit parsing field numbers.
//
// This requires careful thought as the MarshalOptions.EmitUnknown
// option allows formatting unknown fields as the field number and the
// best-effort textual representation of the field value. In that case,
// it may not be possible to unmarshal the value from a parser that does
// have information about the unknown field.
return d.newError(tok.Pos(), "cannot specify field by number: %v", tok.RawString())
}
switch {
case fd.IsList():
kind := fd.Kind()
if kind != protoreflect.MessageKind && kind != protoreflect.GroupKind && !tok.HasSeparator() {
return d.syntaxError(tok.Pos(), "missing field separator :")
}
list := m.Mutable(fd).List()
if err := d.unmarshalList(fd, list); err != nil {
return err
}
case fd.IsMap():
mmap := m.Mutable(fd).Map()
if err := d.unmarshalMap(fd, mmap); err != nil {
return err
}
default:
kind := fd.Kind()
if kind != protoreflect.MessageKind && kind != protoreflect.GroupKind && !tok.HasSeparator() {
return d.syntaxError(tok.Pos(), "missing field separator :")
}
// If field is a oneof, check if it has already been set.
if od := fd.ContainingOneof(); od != nil {
idx := uint64(od.Index())
if seenOneofs.Has(idx) {
return d.newError(tok.Pos(), "error parsing %q, oneof %v is already set", tok.RawString(), od.FullName())
}
seenOneofs.Set(idx)
}
num := uint64(fd.Number())
if seenNums.Has(num) {
return d.newError(tok.Pos(), "non-repeated field %q is repeated", tok.RawString())
}
if err := d.unmarshalSingular(fd, m); err != nil {
return err
}
seenNums.Set(num)
}
}
return nil
}
// unmarshalSingular unmarshals a non-repeated field value specified by the
// given FieldDescriptor.
func (d decoder) unmarshalSingular(fd protoreflect.FieldDescriptor, m protoreflect.Message) error {
var val protoreflect.Value
var err error
switch fd.Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
val = m.NewField(fd)
err = d.unmarshalMessage(val.Message(), true)
default:
val, err = d.unmarshalScalar(fd)
}
if err == nil {
m.Set(fd, val)
}
return err
}
// unmarshalScalar unmarshals a scalar/enum protoreflect.Value specified by the
// given FieldDescriptor.
func (d decoder) unmarshalScalar(fd protoreflect.FieldDescriptor) (protoreflect.Value, error) {
tok, err := d.Read()
if err != nil {
return protoreflect.Value{}, err
}
if tok.Kind() != text.Scalar {
return protoreflect.Value{}, d.unexpectedTokenError(tok)
}
kind := fd.Kind()
switch kind {
case protoreflect.BoolKind:
if b, ok := tok.Bool(); ok {
return protoreflect.ValueOfBool(b), nil
}
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
if n, ok := tok.Int32(); ok {
return protoreflect.ValueOfInt32(n), nil
}
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
if n, ok := tok.Int64(); ok {
return protoreflect.ValueOfInt64(n), nil
}
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
if n, ok := tok.Uint32(); ok {
return protoreflect.ValueOfUint32(n), nil
}
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
if n, ok := tok.Uint64(); ok {
return protoreflect.ValueOfUint64(n), nil
}
case protoreflect.FloatKind:
if n, ok := tok.Float32(); ok {
return protoreflect.ValueOfFloat32(n), nil
}
case protoreflect.DoubleKind:
if n, ok := tok.Float64(); ok {
return protoreflect.ValueOfFloat64(n), nil
}
case protoreflect.StringKind:
if s, ok := tok.String(); ok {
if strs.EnforceUTF8(fd) && !utf8.ValidString(s) {
return protoreflect.Value{}, d.newError(tok.Pos(), "contains invalid UTF-8")
}
return protoreflect.ValueOfString(s), nil
}
case protoreflect.BytesKind:
if b, ok := tok.String(); ok {
return protoreflect.ValueOfBytes([]byte(b)), nil
}
case protoreflect.EnumKind:
if lit, ok := tok.Enum(); ok {
// Lookup EnumNumber based on name.
if enumVal := fd.Enum().Values().ByName(protoreflect.Name(lit)); enumVal != nil {
return protoreflect.ValueOfEnum(enumVal.Number()), nil
}
}
if num, ok := tok.Int32(); ok {
return protoreflect.ValueOfEnum(protoreflect.EnumNumber(num)), nil
}
default:
panic(fmt.Sprintf("invalid scalar kind %v", kind))
}
return protoreflect.Value{}, d.newError(tok.Pos(), "invalid value for %v type: %v", kind, tok.RawString())
}
// unmarshalList unmarshals into given protoreflect.List. A list value can
// either be in [] syntax or simply just a single scalar/message value.
func (d decoder) unmarshalList(fd protoreflect.FieldDescriptor, list protoreflect.List) error {
tok, err := d.Peek()
if err != nil {
return err
}
switch fd.Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
switch tok.Kind() {
case text.ListOpen:
d.Read()
for {
tok, err := d.Peek()
if err != nil {
return err
}
switch tok.Kind() {
case text.ListClose:
d.Read()
return nil
case text.MessageOpen:
pval := list.NewElement()
if err := d.unmarshalMessage(pval.Message(), true); err != nil {
return err
}
list.Append(pval)
default:
return d.unexpectedTokenError(tok)
}
}
case text.MessageOpen:
pval := list.NewElement()
if err := d.unmarshalMessage(pval.Message(), true); err != nil {
return err
}
list.Append(pval)
return nil
}
default:
switch tok.Kind() {
case text.ListOpen:
d.Read()
for {
tok, err := d.Peek()
if err != nil {
return err
}
switch tok.Kind() {
case text.ListClose:
d.Read()
return nil
case text.Scalar:
pval, err := d.unmarshalScalar(fd)
if err != nil {
return err
}
list.Append(pval)
default:
return d.unexpectedTokenError(tok)
}
}
case text.Scalar:
pval, err := d.unmarshalScalar(fd)
if err != nil {
return err
}
list.Append(pval)
return nil
}
}
return d.unexpectedTokenError(tok)
}
// unmarshalMap unmarshals into given protoreflect.Map. A map value is a
// textproto message containing {key: <kvalue>, value: <mvalue>}.
func (d decoder) unmarshalMap(fd protoreflect.FieldDescriptor, mmap protoreflect.Map) error {
// Determine ahead whether map entry is a scalar type or a message type in
// order to call the appropriate unmarshalMapValue func inside
// unmarshalMapEntry.
var unmarshalMapValue func() (protoreflect.Value, error)
switch fd.MapValue().Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
unmarshalMapValue = func() (protoreflect.Value, error) {
pval := mmap.NewValue()
if err := d.unmarshalMessage(pval.Message(), true); err != nil {
return protoreflect.Value{}, err
}
return pval, nil
}
default:
unmarshalMapValue = func() (protoreflect.Value, error) {
return d.unmarshalScalar(fd.MapValue())
}
}
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case text.MessageOpen:
return d.unmarshalMapEntry(fd, mmap, unmarshalMapValue)
case text.ListOpen:
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case text.ListClose:
return nil
case text.MessageOpen:
if err := d.unmarshalMapEntry(fd, mmap, unmarshalMapValue); err != nil {
return err
}
default:
return d.unexpectedTokenError(tok)
}
}
default:
return d.unexpectedTokenError(tok)
}
}
// unmarshalMap unmarshals into given protoreflect.Map. A map value is a
// textproto message containing {key: <kvalue>, value: <mvalue>}.
func (d decoder) unmarshalMapEntry(fd protoreflect.FieldDescriptor, mmap protoreflect.Map, unmarshalMapValue func() (protoreflect.Value, error)) error {
var key protoreflect.MapKey
var pval protoreflect.Value
Loop:
for {
// Read field name.
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case text.Name:
if tok.NameKind() != text.IdentName {
if !d.opts.DiscardUnknown {
return d.newError(tok.Pos(), "unknown map entry field %q", tok.RawString())
}
d.skipValue()
continue Loop
}
// Continue below.
case text.MessageClose:
break Loop
default:
return d.unexpectedTokenError(tok)
}
switch name := protoreflect.Name(tok.IdentName()); name {
case genid.MapEntry_Key_field_name:
if !tok.HasSeparator() {
return d.syntaxError(tok.Pos(), "missing field separator :")
}
if key.IsValid() {
return d.newError(tok.Pos(), "map entry %q cannot be repeated", name)
}
val, err := d.unmarshalScalar(fd.MapKey())
if err != nil {
return err
}
key = val.MapKey()
case genid.MapEntry_Value_field_name:
if kind := fd.MapValue().Kind(); (kind != protoreflect.MessageKind) && (kind != protoreflect.GroupKind) {
if !tok.HasSeparator() {
return d.syntaxError(tok.Pos(), "missing field separator :")
}
}
if pval.IsValid() {
return d.newError(tok.Pos(), "map entry %q cannot be repeated", name)
}
pval, err = unmarshalMapValue()
if err != nil {
return err
}
default:
if !d.opts.DiscardUnknown {
return d.newError(tok.Pos(), "unknown map entry field %q", name)
}
d.skipValue()
}
}
if !key.IsValid() {
key = fd.MapKey().Default().MapKey()
}
if !pval.IsValid() {
switch fd.MapValue().Kind() {
case protoreflect.MessageKind, protoreflect.GroupKind:
// If value field is not set for message/group types, construct an
// empty one as default.
pval = mmap.NewValue()
default:
pval = fd.MapValue().Default()
}
}
mmap.Set(key, pval)
return nil
}
// unmarshalAny unmarshals an Any textproto. It can either be in expanded form
// or non-expanded form.
func (d decoder) unmarshalAny(m protoreflect.Message, checkDelims bool) error {
var typeURL string
var bValue []byte
var seenTypeUrl bool
var seenValue bool
var isExpanded bool
if checkDelims {
tok, err := d.Read()
if err != nil {
return err
}
if tok.Kind() != text.MessageOpen {
return d.unexpectedTokenError(tok)
}
}
Loop:
for {
// Read field name. Can only have 3 possible field names, i.e. type_url,
// value and type URL name inside [].
tok, err := d.Read()
if err != nil {
return err
}
if typ := tok.Kind(); typ != text.Name {
if checkDelims {
if typ == text.MessageClose {
break Loop
}
} else if typ == text.EOF {
break Loop
}
return d.unexpectedTokenError(tok)
}
switch tok.NameKind() {
case text.IdentName:
// Both type_url and value fields require field separator :.
if !tok.HasSeparator() {
return d.syntaxError(tok.Pos(), "missing field separator :")
}
switch name := protoreflect.Name(tok.IdentName()); name {
case genid.Any_TypeUrl_field_name:
if seenTypeUrl {
return d.newError(tok.Pos(), "duplicate %v field", genid.Any_TypeUrl_field_fullname)
}
if isExpanded {
return d.newError(tok.Pos(), "conflict with [%s] field", typeURL)
}
tok, err := d.Read()
if err != nil {
return err
}
var ok bool
typeURL, ok = tok.String()
if !ok {
return d.newError(tok.Pos(), "invalid %v field value: %v", genid.Any_TypeUrl_field_fullname, tok.RawString())
}
seenTypeUrl = true
case genid.Any_Value_field_name:
if seenValue {
return d.newError(tok.Pos(), "duplicate %v field", genid.Any_Value_field_fullname)
}
if isExpanded {
return d.newError(tok.Pos(), "conflict with [%s] field", typeURL)
}
tok, err := d.Read()
if err != nil {
return err
}
s, ok := tok.String()
if !ok {
return d.newError(tok.Pos(), "invalid %v field value: %v", genid.Any_Value_field_fullname, tok.RawString())
}
bValue = []byte(s)
seenValue = true
default:
if !d.opts.DiscardUnknown {
return d.newError(tok.Pos(), "invalid field name %q in %v message", tok.RawString(), genid.Any_message_fullname)
}
}
case text.TypeName:
if isExpanded {
return d.newError(tok.Pos(), "cannot have more than one type")
}
if seenTypeUrl {
return d.newError(tok.Pos(), "conflict with type_url field")
}
typeURL = tok.TypeName()
var err error
bValue, err = d.unmarshalExpandedAny(typeURL, tok.Pos())
if err != nil {
return err
}
isExpanded = true
default:
if !d.opts.DiscardUnknown {
return d.newError(tok.Pos(), "invalid field name %q in %v message", tok.RawString(), genid.Any_message_fullname)
}
}
}
fds := m.Descriptor().Fields()
if len(typeURL) > 0 {
m.Set(fds.ByNumber(genid.Any_TypeUrl_field_number), protoreflect.ValueOfString(typeURL))
}
if len(bValue) > 0 {
m.Set(fds.ByNumber(genid.Any_Value_field_number), protoreflect.ValueOfBytes(bValue))
}
return nil
}
func (d decoder) unmarshalExpandedAny(typeURL string, pos int) ([]byte, error) {
mt, err := d.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
return nil, d.newError(pos, "unable to resolve message [%v]: %v", typeURL, err)
}
// Create new message for the embedded message type and unmarshal the value
// field into it.
m := mt.New()
if err := d.unmarshalMessage(m, true); err != nil {
return nil, err
}
// Serialize the embedded message and return the resulting bytes.
b, err := proto.MarshalOptions{
AllowPartial: true, // Never check required fields inside an Any.
Deterministic: true,
}.Marshal(m.Interface())
if err != nil {
return nil, d.newError(pos, "error in marshaling message into Any.value: %v", err)
}
return b, nil
}
// skipValue makes the decoder parse a field value in order to advance the read
// to the next field. It relies on Read returning an error if the types are not
// in valid sequence.
func (d decoder) skipValue() error {
tok, err := d.Read()
if err != nil {
return err
}
// Only need to continue reading for messages and lists.
switch tok.Kind() {
case text.MessageOpen:
return d.skipMessageValue()
case text.ListOpen:
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case text.ListClose:
return nil
case text.MessageOpen:
if err := d.skipMessageValue(); err != nil {
return err
}
default:
// Skip items. This will not validate whether skipped values are
// of the same type or not, same behavior as C++
// TextFormat::Parser::AllowUnknownField(true) version 3.8.0.
}
}
}
return nil
}
// skipMessageValue makes the decoder parse and skip over all fields in a
// message. It assumes that the previous read type is MessageOpen.
func (d decoder) skipMessageValue() error {
for {
tok, err := d.Read()
if err != nil {
return err
}
switch tok.Kind() {
case text.MessageClose:
return nil
case text.Name:
if err := d.skipValue(); err != nil {
return err
}
}
}
}

View File

@ -0,0 +1,7 @@
// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package prototext marshals and unmarshals protocol buffer messages as the
// textproto format.
package prototext

View File

@ -0,0 +1,380 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package prototext
import (
"fmt"
"strconv"
"unicode/utf8"
"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/internal/encoding/messageset"
"google.golang.org/protobuf/internal/encoding/text"
"google.golang.org/protobuf/internal/errors"
"google.golang.org/protobuf/internal/flags"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/internal/order"
"google.golang.org/protobuf/internal/pragma"
"google.golang.org/protobuf/internal/strs"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/reflect/protoregistry"
)
const defaultIndent = " "
// Format formats the message as a multiline string.
// This function is only intended for human consumption and ignores errors.
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func Format(m proto.Message) string {
return MarshalOptions{Multiline: true}.Format(m)
}
// Marshal writes the given [proto.Message] in textproto format using default
// options. Do not depend on the output being stable. Its output will change
// across different builds of your program, even when using the same version of
// the protobuf module.
func Marshal(m proto.Message) ([]byte, error) {
return MarshalOptions{}.Marshal(m)
}
// MarshalOptions is a configurable text format marshaler.
type MarshalOptions struct {
pragma.NoUnkeyedLiterals
// Multiline specifies whether the marshaler should format the output in
// indented-form with every textual element on a new line.
// If Indent is an empty string, then an arbitrary indent is chosen.
Multiline bool
// Indent specifies the set of indentation characters to use in a multiline
// formatted output such that every entry is preceded by Indent and
// terminated by a newline. If non-empty, then Multiline is treated as true.
// Indent can only be composed of space or tab characters.
Indent string
// EmitASCII specifies whether to format strings and bytes as ASCII only
// as opposed to using UTF-8 encoding when possible.
EmitASCII bool
// allowInvalidUTF8 specifies whether to permit the encoding of strings
// with invalid UTF-8. This is unexported as it is intended to only
// be specified by the Format method.
allowInvalidUTF8 bool
// AllowPartial allows messages that have missing required fields to marshal
// without returning an error. If AllowPartial is false (the default),
// Marshal will return error if there are any missing required fields.
AllowPartial bool
// EmitUnknown specifies whether to emit unknown fields in the output.
// If specified, the unmarshaler may be unable to parse the output.
// The default is to exclude unknown fields.
EmitUnknown bool
// Resolver is used for looking up types when expanding google.protobuf.Any
// messages. If nil, this defaults to using protoregistry.GlobalTypes.
Resolver interface {
protoregistry.ExtensionTypeResolver
protoregistry.MessageTypeResolver
}
}
// Format formats the message as a string.
// This method is only intended for human consumption and ignores errors.
// Do not depend on the output being stable. Its output will change across
// different builds of your program, even when using the same version of the
// protobuf module.
func (o MarshalOptions) Format(m proto.Message) string {
if m == nil || !m.ProtoReflect().IsValid() {
return "<nil>" // invalid syntax, but okay since this is for debugging
}
o.allowInvalidUTF8 = true
o.AllowPartial = true
o.EmitUnknown = true
b, _ := o.Marshal(m)
return string(b)
}
// Marshal writes the given [proto.Message] in textproto format using options in
// MarshalOptions object. Do not depend on the output being stable. Its output
// will change across different builds of your program, even when using the
// same version of the protobuf module.
func (o MarshalOptions) Marshal(m proto.Message) ([]byte, error) {
return o.marshal(nil, m)
}
// MarshalAppend appends the textproto format encoding of m to b,
// returning the result.
func (o MarshalOptions) MarshalAppend(b []byte, m proto.Message) ([]byte, error) {
return o.marshal(b, m)
}
// marshal is a centralized function that all marshal operations go through.
// For profiling purposes, avoid changing the name of this function or
// introducing other code paths for marshal that do not go through this.
func (o MarshalOptions) marshal(b []byte, m proto.Message) ([]byte, error) {
var delims = [2]byte{'{', '}'}
if o.Multiline && o.Indent == "" {
o.Indent = defaultIndent
}
if o.Resolver == nil {
o.Resolver = protoregistry.GlobalTypes
}
internalEnc, err := text.NewEncoder(b, o.Indent, delims, o.EmitASCII)
if err != nil {
return nil, err
}
// Treat nil message interface as an empty message,
// in which case there is nothing to output.
if m == nil {
return b, nil
}
enc := encoder{internalEnc, o}
err = enc.marshalMessage(m.ProtoReflect(), false)
if err != nil {
return nil, err
}
out := enc.Bytes()
if len(o.Indent) > 0 && len(out) > 0 {
out = append(out, '\n')
}
if o.AllowPartial {
return out, nil
}
return out, proto.CheckInitialized(m)
}
type encoder struct {
*text.Encoder
opts MarshalOptions
}
// marshalMessage marshals the given protoreflect.Message.
func (e encoder) marshalMessage(m protoreflect.Message, inclDelims bool) error {
messageDesc := m.Descriptor()
if !flags.ProtoLegacy && messageset.IsMessageSet(messageDesc) {
return errors.New("no support for proto1 MessageSets")
}
if inclDelims {
e.StartMessage()
defer e.EndMessage()
}
// Handle Any expansion.
if messageDesc.FullName() == genid.Any_message_fullname {
if e.marshalAny(m) {
return nil
}
// If unable to expand, continue on to marshal Any as a regular message.
}
// Marshal fields.
var err error
order.RangeFields(m, order.IndexNameFieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
if err = e.marshalField(fd.TextName(), v, fd); err != nil {
return false
}
return true
})
if err != nil {
return err
}
// Marshal unknown fields.
if e.opts.EmitUnknown {
e.marshalUnknown(m.GetUnknown())
}
return nil
}
// marshalField marshals the given field with protoreflect.Value.
func (e encoder) marshalField(name string, val protoreflect.Value, fd protoreflect.FieldDescriptor) error {
switch {
case fd.IsList():
return e.marshalList(name, val.List(), fd)
case fd.IsMap():
return e.marshalMap(name, val.Map(), fd)
default:
e.WriteName(name)
return e.marshalSingular(val, fd)
}
}
// marshalSingular marshals the given non-repeated field value. This includes
// all scalar types, enums, messages, and groups.
func (e encoder) marshalSingular(val protoreflect.Value, fd protoreflect.FieldDescriptor) error {
kind := fd.Kind()
switch kind {
case protoreflect.BoolKind:
e.WriteBool(val.Bool())
case protoreflect.StringKind:
s := val.String()
if !e.opts.allowInvalidUTF8 && strs.EnforceUTF8(fd) && !utf8.ValidString(s) {
return errors.InvalidUTF8(string(fd.FullName()))
}
e.WriteString(s)
case protoreflect.Int32Kind, protoreflect.Int64Kind,
protoreflect.Sint32Kind, protoreflect.Sint64Kind,
protoreflect.Sfixed32Kind, protoreflect.Sfixed64Kind:
e.WriteInt(val.Int())
case protoreflect.Uint32Kind, protoreflect.Uint64Kind,
protoreflect.Fixed32Kind, protoreflect.Fixed64Kind:
e.WriteUint(val.Uint())
case protoreflect.FloatKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
e.WriteFloat(val.Float(), 32)
case protoreflect.DoubleKind:
// Encoder.WriteFloat handles the special numbers NaN and infinites.
e.WriteFloat(val.Float(), 64)
case protoreflect.BytesKind:
e.WriteString(string(val.Bytes()))
case protoreflect.EnumKind:
num := val.Enum()
if desc := fd.Enum().Values().ByNumber(num); desc != nil {
e.WriteLiteral(string(desc.Name()))
} else {
// Use numeric value if there is no enum description.
e.WriteInt(int64(num))
}
case protoreflect.MessageKind, protoreflect.GroupKind:
return e.marshalMessage(val.Message(), true)
default:
panic(fmt.Sprintf("%v has unknown kind: %v", fd.FullName(), kind))
}
return nil
}
// marshalList marshals the given protoreflect.List as multiple name-value fields.
func (e encoder) marshalList(name string, list protoreflect.List, fd protoreflect.FieldDescriptor) error {
size := list.Len()
for i := 0; i < size; i++ {
e.WriteName(name)
if err := e.marshalSingular(list.Get(i), fd); err != nil {
return err
}
}
return nil
}
// marshalMap marshals the given protoreflect.Map as multiple name-value fields.
func (e encoder) marshalMap(name string, mmap protoreflect.Map, fd protoreflect.FieldDescriptor) error {
var err error
order.RangeEntries(mmap, order.GenericKeyOrder, func(key protoreflect.MapKey, val protoreflect.Value) bool {
e.WriteName(name)
e.StartMessage()
defer e.EndMessage()
e.WriteName(string(genid.MapEntry_Key_field_name))
err = e.marshalSingular(key.Value(), fd.MapKey())
if err != nil {
return false
}
e.WriteName(string(genid.MapEntry_Value_field_name))
err = e.marshalSingular(val, fd.MapValue())
if err != nil {
return false
}
return true
})
return err
}
// marshalUnknown parses the given []byte and marshals fields out.
// This function assumes proper encoding in the given []byte.
func (e encoder) marshalUnknown(b []byte) {
const dec = 10
const hex = 16
for len(b) > 0 {
num, wtype, n := protowire.ConsumeTag(b)
b = b[n:]
e.WriteName(strconv.FormatInt(int64(num), dec))
switch wtype {
case protowire.VarintType:
var v uint64
v, n = protowire.ConsumeVarint(b)
e.WriteUint(v)
case protowire.Fixed32Type:
var v uint32
v, n = protowire.ConsumeFixed32(b)
e.WriteLiteral("0x" + strconv.FormatUint(uint64(v), hex))
case protowire.Fixed64Type:
var v uint64
v, n = protowire.ConsumeFixed64(b)
e.WriteLiteral("0x" + strconv.FormatUint(v, hex))
case protowire.BytesType:
var v []byte
v, n = protowire.ConsumeBytes(b)
e.WriteString(string(v))
case protowire.StartGroupType:
e.StartMessage()
var v []byte
v, n = protowire.ConsumeGroup(num, b)
e.marshalUnknown(v)
e.EndMessage()
default:
panic(fmt.Sprintf("prototext: error parsing unknown field wire type: %v", wtype))
}
b = b[n:]
}
}
// marshalAny marshals the given google.protobuf.Any message in expanded form.
// It returns true if it was able to marshal, else false.
func (e encoder) marshalAny(any protoreflect.Message) bool {
// Construct the embedded message.
fds := any.Descriptor().Fields()
fdType := fds.ByNumber(genid.Any_TypeUrl_field_number)
typeURL := any.Get(fdType).String()
mt, err := e.opts.Resolver.FindMessageByURL(typeURL)
if err != nil {
return false
}
m := mt.New().Interface()
// Unmarshal bytes into embedded message.
fdValue := fds.ByNumber(genid.Any_Value_field_number)
value := any.Get(fdValue)
err = proto.UnmarshalOptions{
AllowPartial: true,
Resolver: e.opts.Resolver,
}.Unmarshal(value.Bytes(), m)
if err != nil {
return false
}
// Get current encoder position. If marshaling fails, reset encoder output
// back to this position.
pos := e.Snapshot()
// Field name is the proto field name enclosed in [].
e.WriteName("[" + typeURL + "]")
err = e.marshalMessage(m.ProtoReflect(), true)
if err != nil {
e.Reset(pos)
return false
}
return true
}

View File

@ -0,0 +1,547 @@
// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package protowire parses and formats the raw wire encoding.
// See https://protobuf.dev/programming-guides/encoding.
//
// For marshaling and unmarshaling entire protobuf messages,
// use the [google.golang.org/protobuf/proto] package instead.
package protowire
import (
"io"
"math"
"math/bits"
"google.golang.org/protobuf/internal/errors"
)
// Number represents the field number.
type Number int32
const (
MinValidNumber Number = 1
FirstReservedNumber Number = 19000
LastReservedNumber Number = 19999
MaxValidNumber Number = 1<<29 - 1
DefaultRecursionLimit = 10000
)
// IsValid reports whether the field number is semantically valid.
func (n Number) IsValid() bool {
return MinValidNumber <= n && n <= MaxValidNumber
}
// Type represents the wire type.
type Type int8
const (
VarintType Type = 0
Fixed32Type Type = 5
Fixed64Type Type = 1
BytesType Type = 2
StartGroupType Type = 3
EndGroupType Type = 4
)
const (
_ = -iota
errCodeTruncated
errCodeFieldNumber
errCodeOverflow
errCodeReserved
errCodeEndGroup
errCodeRecursionDepth
)
var (
errFieldNumber = errors.New("invalid field number")
errOverflow = errors.New("variable length integer overflow")
errReserved = errors.New("cannot parse reserved wire type")
errEndGroup = errors.New("mismatching end group marker")
errParse = errors.New("parse error")
)
// ParseError converts an error code into an error value.
// This returns nil if n is a non-negative number.
func ParseError(n int) error {
if n >= 0 {
return nil
}
switch n {
case errCodeTruncated:
return io.ErrUnexpectedEOF
case errCodeFieldNumber:
return errFieldNumber
case errCodeOverflow:
return errOverflow
case errCodeReserved:
return errReserved
case errCodeEndGroup:
return errEndGroup
default:
return errParse
}
}
// ConsumeField parses an entire field record (both tag and value) and returns
// the field number, the wire type, and the total length.
// This returns a negative length upon an error (see [ParseError]).
//
// The total length includes the tag header and the end group marker (if the
// field is a group).
func ConsumeField(b []byte) (Number, Type, int) {
num, typ, n := ConsumeTag(b)
if n < 0 {
return 0, 0, n // forward error code
}
m := ConsumeFieldValue(num, typ, b[n:])
if m < 0 {
return 0, 0, m // forward error code
}
return num, typ, n + m
}
// ConsumeFieldValue parses a field value and returns its length.
// This assumes that the field [Number] and wire [Type] have already been parsed.
// This returns a negative length upon an error (see [ParseError]).
//
// When parsing a group, the length includes the end group marker and
// the end group is verified to match the starting field number.
func ConsumeFieldValue(num Number, typ Type, b []byte) (n int) {
return consumeFieldValueD(num, typ, b, DefaultRecursionLimit)
}
func consumeFieldValueD(num Number, typ Type, b []byte, depth int) (n int) {
switch typ {
case VarintType:
_, n = ConsumeVarint(b)
return n
case Fixed32Type:
_, n = ConsumeFixed32(b)
return n
case Fixed64Type:
_, n = ConsumeFixed64(b)
return n
case BytesType:
_, n = ConsumeBytes(b)
return n
case StartGroupType:
if depth < 0 {
return errCodeRecursionDepth
}
n0 := len(b)
for {
num2, typ2, n := ConsumeTag(b)
if n < 0 {
return n // forward error code
}
b = b[n:]
if typ2 == EndGroupType {
if num != num2 {
return errCodeEndGroup
}
return n0 - len(b)
}
n = consumeFieldValueD(num2, typ2, b, depth-1)
if n < 0 {
return n // forward error code
}
b = b[n:]
}
case EndGroupType:
return errCodeEndGroup
default:
return errCodeReserved
}
}
// AppendTag encodes num and typ as a varint-encoded tag and appends it to b.
func AppendTag(b []byte, num Number, typ Type) []byte {
return AppendVarint(b, EncodeTag(num, typ))
}
// ConsumeTag parses b as a varint-encoded tag, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeTag(b []byte) (Number, Type, int) {
v, n := ConsumeVarint(b)
if n < 0 {
return 0, 0, n // forward error code
}
num, typ := DecodeTag(v)
if num < MinValidNumber {
return 0, 0, errCodeFieldNumber
}
return num, typ, n
}
func SizeTag(num Number) int {
return SizeVarint(EncodeTag(num, 0)) // wire type has no effect on size
}
// AppendVarint appends v to b as a varint-encoded uint64.
func AppendVarint(b []byte, v uint64) []byte {
switch {
case v < 1<<7:
b = append(b, byte(v))
case v < 1<<14:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte(v>>7))
case v < 1<<21:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte(v>>14))
case v < 1<<28:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte(v>>21))
case v < 1<<35:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte(v>>28))
case v < 1<<42:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte((v>>28)&0x7f|0x80),
byte(v>>35))
case v < 1<<49:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte((v>>28)&0x7f|0x80),
byte((v>>35)&0x7f|0x80),
byte(v>>42))
case v < 1<<56:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte((v>>28)&0x7f|0x80),
byte((v>>35)&0x7f|0x80),
byte((v>>42)&0x7f|0x80),
byte(v>>49))
case v < 1<<63:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte((v>>28)&0x7f|0x80),
byte((v>>35)&0x7f|0x80),
byte((v>>42)&0x7f|0x80),
byte((v>>49)&0x7f|0x80),
byte(v>>56))
default:
b = append(b,
byte((v>>0)&0x7f|0x80),
byte((v>>7)&0x7f|0x80),
byte((v>>14)&0x7f|0x80),
byte((v>>21)&0x7f|0x80),
byte((v>>28)&0x7f|0x80),
byte((v>>35)&0x7f|0x80),
byte((v>>42)&0x7f|0x80),
byte((v>>49)&0x7f|0x80),
byte((v>>56)&0x7f|0x80),
1)
}
return b
}
// ConsumeVarint parses b as a varint-encoded uint64, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeVarint(b []byte) (v uint64, n int) {
var y uint64
if len(b) <= 0 {
return 0, errCodeTruncated
}
v = uint64(b[0])
if v < 0x80 {
return v, 1
}
v -= 0x80
if len(b) <= 1 {
return 0, errCodeTruncated
}
y = uint64(b[1])
v += y << 7
if y < 0x80 {
return v, 2
}
v -= 0x80 << 7
if len(b) <= 2 {
return 0, errCodeTruncated
}
y = uint64(b[2])
v += y << 14
if y < 0x80 {
return v, 3
}
v -= 0x80 << 14
if len(b) <= 3 {
return 0, errCodeTruncated
}
y = uint64(b[3])
v += y << 21
if y < 0x80 {
return v, 4
}
v -= 0x80 << 21
if len(b) <= 4 {
return 0, errCodeTruncated
}
y = uint64(b[4])
v += y << 28
if y < 0x80 {
return v, 5
}
v -= 0x80 << 28
if len(b) <= 5 {
return 0, errCodeTruncated
}
y = uint64(b[5])
v += y << 35
if y < 0x80 {
return v, 6
}
v -= 0x80 << 35
if len(b) <= 6 {
return 0, errCodeTruncated
}
y = uint64(b[6])
v += y << 42
if y < 0x80 {
return v, 7
}
v -= 0x80 << 42
if len(b) <= 7 {
return 0, errCodeTruncated
}
y = uint64(b[7])
v += y << 49
if y < 0x80 {
return v, 8
}
v -= 0x80 << 49
if len(b) <= 8 {
return 0, errCodeTruncated
}
y = uint64(b[8])
v += y << 56
if y < 0x80 {
return v, 9
}
v -= 0x80 << 56
if len(b) <= 9 {
return 0, errCodeTruncated
}
y = uint64(b[9])
v += y << 63
if y < 2 {
return v, 10
}
return 0, errCodeOverflow
}
// SizeVarint returns the encoded size of a varint.
// The size is guaranteed to be within 1 and 10, inclusive.
func SizeVarint(v uint64) int {
// This computes 1 + (bits.Len64(v)-1)/7.
// 9/64 is a good enough approximation of 1/7
return int(9*uint32(bits.Len64(v))+64) / 64
}
// AppendFixed32 appends v to b as a little-endian uint32.
func AppendFixed32(b []byte, v uint32) []byte {
return append(b,
byte(v>>0),
byte(v>>8),
byte(v>>16),
byte(v>>24))
}
// ConsumeFixed32 parses b as a little-endian uint32, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeFixed32(b []byte) (v uint32, n int) {
if len(b) < 4 {
return 0, errCodeTruncated
}
v = uint32(b[0])<<0 | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
return v, 4
}
// SizeFixed32 returns the encoded size of a fixed32; which is always 4.
func SizeFixed32() int {
return 4
}
// AppendFixed64 appends v to b as a little-endian uint64.
func AppendFixed64(b []byte, v uint64) []byte {
return append(b,
byte(v>>0),
byte(v>>8),
byte(v>>16),
byte(v>>24),
byte(v>>32),
byte(v>>40),
byte(v>>48),
byte(v>>56))
}
// ConsumeFixed64 parses b as a little-endian uint64, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeFixed64(b []byte) (v uint64, n int) {
if len(b) < 8 {
return 0, errCodeTruncated
}
v = uint64(b[0])<<0 | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
return v, 8
}
// SizeFixed64 returns the encoded size of a fixed64; which is always 8.
func SizeFixed64() int {
return 8
}
// AppendBytes appends v to b as a length-prefixed bytes value.
func AppendBytes(b []byte, v []byte) []byte {
return append(AppendVarint(b, uint64(len(v))), v...)
}
// ConsumeBytes parses b as a length-prefixed bytes value, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeBytes(b []byte) (v []byte, n int) {
m, n := ConsumeVarint(b)
if n < 0 {
return nil, n // forward error code
}
if m > uint64(len(b[n:])) {
return nil, errCodeTruncated
}
return b[n:][:m], n + int(m)
}
// SizeBytes returns the encoded size of a length-prefixed bytes value,
// given only the length.
func SizeBytes(n int) int {
return SizeVarint(uint64(n)) + n
}
// AppendString appends v to b as a length-prefixed bytes value.
func AppendString(b []byte, v string) []byte {
return append(AppendVarint(b, uint64(len(v))), v...)
}
// ConsumeString parses b as a length-prefixed bytes value, reporting its length.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeString(b []byte) (v string, n int) {
bb, n := ConsumeBytes(b)
return string(bb), n
}
// AppendGroup appends v to b as group value, with a trailing end group marker.
// The value v must not contain the end marker.
func AppendGroup(b []byte, num Number, v []byte) []byte {
return AppendVarint(append(b, v...), EncodeTag(num, EndGroupType))
}
// ConsumeGroup parses b as a group value until the trailing end group marker,
// and verifies that the end marker matches the provided num. The value v
// does not contain the end marker, while the length does contain the end marker.
// This returns a negative length upon an error (see [ParseError]).
func ConsumeGroup(num Number, b []byte) (v []byte, n int) {
n = ConsumeFieldValue(num, StartGroupType, b)
if n < 0 {
return nil, n // forward error code
}
b = b[:n]
// Truncate off end group marker, but need to handle denormalized varints.
// Assuming end marker is never 0 (which is always the case since
// EndGroupType is non-zero), we can truncate all trailing bytes where the
// lower 7 bits are all zero (implying that the varint is denormalized).
for len(b) > 0 && b[len(b)-1]&0x7f == 0 {
b = b[:len(b)-1]
}
b = b[:len(b)-SizeTag(num)]
return b, n
}
// SizeGroup returns the encoded size of a group, given only the length.
func SizeGroup(num Number, n int) int {
return n + SizeTag(num)
}
// DecodeTag decodes the field [Number] and wire [Type] from its unified form.
// The [Number] is -1 if the decoded field number overflows int32.
// Other than overflow, this does not check for field number validity.
func DecodeTag(x uint64) (Number, Type) {
// NOTE: MessageSet allows for larger field numbers than normal.
if x>>3 > uint64(math.MaxInt32) {
return -1, 0
}
return Number(x >> 3), Type(x & 7)
}
// EncodeTag encodes the field [Number] and wire [Type] into its unified form.
func EncodeTag(num Number, typ Type) uint64 {
return uint64(num)<<3 | uint64(typ&7)
}
// DecodeZigZag decodes a zig-zag-encoded uint64 as an int64.
//
// Input: {…, 5, 3, 1, 0, 2, 4, 6, …}
// Output: {…, -3, -2, -1, 0, +1, +2, +3, …}
func DecodeZigZag(x uint64) int64 {
return int64(x>>1) ^ int64(x)<<63>>63
}
// EncodeZigZag encodes an int64 as a zig-zag-encoded uint64.
//
// Input: {…, -3, -2, -1, 0, +1, +2, +3, …}
// Output: {…, 5, 3, 1, 0, 2, 4, 6, …}
func EncodeZigZag(x int64) uint64 {
return uint64(x<<1) ^ uint64(x>>63)
}
// DecodeBool decodes a uint64 as a bool.
//
// Input: { 0, 1, 2, …}
// Output: {false, true, true, …}
func DecodeBool(x uint64) bool {
return x != 0
}
// EncodeBool encodes a bool as a uint64.
//
// Input: {false, true}
// Output: { 0, 1}
func EncodeBool(x bool) uint64 {
if x {
return 1
}
return 0
}