rebase: bump github.com/kubernetes-csi/external-snapshotter/client/v6

Bumps [github.com/kubernetes-csi/external-snapshotter/client/v6](https://github.com/kubernetes-csi/external-snapshotter) from 6.1.0 to 6.2.0.
- [Release notes](https://github.com/kubernetes-csi/external-snapshotter/releases)
- [Commits](https://github.com/kubernetes-csi/external-snapshotter/compare/v6.1.0...v6.2.0)

---
updated-dependencies:
- dependency-name: github.com/kubernetes-csi/external-snapshotter/client/v6
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot] 2023-01-09 20:03:41 +00:00 committed by mergify[bot]
parent 544c5f33f3
commit c1a636b431
41 changed files with 10254 additions and 36 deletions

8
go.mod
View File

@ -20,7 +20,7 @@ require (
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/hashicorp/vault/api v1.8.2
github.com/kubernetes-csi/csi-lib-utils v0.11.0
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0
github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a
github.com/onsi/ginkgo/v2 v2.4.0
github.com/onsi/gomega v1.23.0
@ -32,8 +32,8 @@ require (
golang.org/x/sys v0.4.0
google.golang.org/grpc v1.51.0
google.golang.org/protobuf v1.28.1
k8s.io/api v0.25.4
k8s.io/apimachinery v0.25.4
k8s.io/api v0.26.0
k8s.io/apimachinery v0.26.0
k8s.io/client-go v12.0.0+incompatible
k8s.io/cloud-provider v0.25.4
k8s.io/klog/v2 v2.80.1
@ -163,7 +163,7 @@ require (
k8s.io/apiserver v0.25.4 // indirect
k8s.io/component-base v0.25.4 // indirect
k8s.io/component-helpers v0.25.4 // indirect
k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea // indirect
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/kubectl v0.0.0 // indirect
k8s.io/kubelet v0.0.0 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.33 // indirect

8
go.sum
View File

@ -737,8 +737,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes-csi/csi-lib-utils v0.11.0 h1:FHWOBtAZBA/hVk7v/qaXgG9Sxv0/n06DebPFuDwumqg=
github.com/kubernetes-csi/csi-lib-utils v0.11.0/go.mod h1:BmGZZB16L18+9+Lgg9YWwBKfNEHIDdgGfAyuW6p2NV0=
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0 h1:yeuon3bOuOADwiWl2CyYrU4vbmYbAzGLCTscE1yLNHk=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0/go.mod h1:eVY6gNtSrhsblGAqKFDG3CrkCLFAjsDvOpPpt+EaS6k=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 h1:cMM5AB37e9aRGjErygVT6EuBPB6s5a+l95OPERmSlVM=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0/go.mod h1:VQVLCPGDX5l6V5PezjlDXLa+SpCbWSVU7B16cFWVVeE=
github.com/layeh/radius v0.0.0-20190322222518-890bc1058917/go.mod h1:fywZKyu//X7iRzaxLgPWsvc0L26IUpVvE/aeIL2JtIQ=
github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@ -1726,8 +1726,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4=
k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg+zw/JGNrgyZRQR7/m6uWg=
k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU=
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+OGxg8HsuBr/5f6tVAjDu6E=
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc=
k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k=
k8s.io/kubelet v0.25.4 h1:24MmTTQGBHr08UkMYFC/RaLjuiMREM53HfRgJKWRquI=

20
vendor/k8s.io/kube-openapi/pkg/internal/flags.go generated vendored Normal file
View File

@ -0,0 +1,20 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
// Used by tests to selectively disable experimental JSON unmarshaler
var UseOptimizedJSONUnmarshaling bool = true

View File

@ -0,0 +1,3 @@
# This source code refers to The Go Authors for copyright purposes.
# The master list of authors is in the main Go distribution,
# visible at https://tip.golang.org/AUTHORS.

View File

@ -0,0 +1,3 @@
# This source code was written by the Go contributors.
# The master list of contributors is in the main Go distribution,
# visible at https://tip.golang.org/CONTRIBUTORS.

View File

@ -0,0 +1,27 @@
Copyright (c) 2020 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View File

@ -0,0 +1,321 @@
# JSON Serialization (v2)
[![GoDev](https://img.shields.io/static/v1?label=godev&message=reference&color=00add8)](https://pkg.go.dev/github.com/go-json-experiment/json)
[![Build Status](https://github.com/go-json-experiment/json/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/go-json-experiment/json/actions)
This module hosts an experimental implementation of v2 `encoding/json`.
The API is unstable and breaking changes will regularly be made.
Do not depend on this in publicly available modules.
## Goals and objectives
* **Mostly backwards compatible:** If possible, v2 should aim to be _mostly_
compatible with v1 in terms of both API and default behavior to ease migration.
For example, the `Marshal` and `Unmarshal` functions are the most widely used
declarations in the v1 package. It seems sensible for equivalent functionality
in v2 to be named the same and have the same signature.
Behaviorally, we should aim for 95% to 99% backwards compatibility.
We do not aim for 100% compatibility since we want the freedom to break
certain behaviors that are now considered to have been a mistake.
We may provide options that can bring the v2 implementation to 100% compatibility,
but it will not be the default.
* **More flexible:** There is a
[long list of feature requests](https://github.com/golang/go/issues?q=is%3Aissue+is%3Aopen+encoding%2Fjson+in%3Atitle).
We should aim to provide the most flexible features that addresses most usages.
We do not want to over fit the v2 API to handle every possible use case.
Ideally, the features provided should be orthogonal in nature such that
any combination of features results in as few surprising edge cases as possible.
* **More performant:** JSON serialization is widely used and any bit of extra
performance gains will be greatly appreciated. Some rarely used behaviors of v1
may be dropped in favor of better performance. For example,
despite `Encoder` and `Decoder` operating on an `io.Writer` and `io.Reader`,
they do not operate in a truly streaming manner,
leading to a loss in performance. The v2 implementation should aim to be truly
streaming by default (see [#33714](https://golang.org/issue/33714)).
* **Easy to use (hard to misuse):** The v2 API should aim to make
the common case easy and the less common case at least possible.
The API should avoid behavior that goes contrary to user expectation,
which may result in subtle bugs (see [#36225](https://golang.org/issue/36225)).
* **v1 and v2 maintainability:** Since the v1 implementation must stay forever,
it would be beneficial if v1 could be implemented under the hood with v2,
allowing for less maintenance burden in the future. This probably implies that
behavioral changes in v2 relative to v1 need to be exposed as options.
* **Avoid unsafe:** Standard library packages generally avoid the use of
package `unsafe` even if it could provide a performance boost.
We aim to preserve this property.
## Expectations
While this module aims to possibly be the v2 implementation of `encoding/json`,
there is no guarantee that this outcome will occur. As with any major change
to the Go standard library, this will eventually go through the
[Go proposal process](https://github.com/golang/proposal#readme).
At the present moment, this is still in the design and experimentation phase
and is not ready for a formal proposal.
There are several possible outcomes from this experiment:
1. We determine that a v2 `encoding/json` would not provide sufficient benefit
over the existing v1 `encoding/json` package. Thus, we abandon this effort.
2. We propose a v2 `encoding/json` design, but it is rejected in favor of some
other design that is considered superior.
3. We propose a v2 `encoding/json` design, but rather than adding an entirely
new v2 `encoding/json` package, we decide to merge its functionality into
the existing v1 `encoding/json` package.
4. We propose a v2 `encoding/json` design and it is accepted, resulting in
its addition to the standard library.
5. Some other unforeseen outcome (among the infinite number of possibilities).
## Development
This module is primarily developed by
[@dsnet](https://github.com/dsnet),
[@mvdan](https://github.com/mvdan), and
[@johanbrandhorst](https://github.com/johanbrandhorst)
with feedback provided by
[@rogpeppe](https://github.com/rogpeppe),
[@ChrisHines](https://github.com/ChrisHines), and
[@rsc](https://github.com/rsc).
Discussion about semantics occur semi-regularly, where a
[record of past meetings can be found here](https://docs.google.com/document/d/1rovrOTd-wTawGMPPlPuKhwXaYBg9VszTXR9AQQL5LfI/edit?usp=sharing).
## Design overview
This package aims to provide a clean separation between syntax and semantics.
Syntax deals with the structural representation of JSON (as specified in
[RFC 4627](https://tools.ietf.org/html/rfc4627),
[RFC 7159](https://tools.ietf.org/html/rfc7159),
[RFC 7493](https://tools.ietf.org/html/rfc7493),
[RFC 8259](https://tools.ietf.org/html/rfc8259), and
[RFC 8785](https://tools.ietf.org/html/rfc8785)).
Semantics deals with the meaning of syntactic data as usable application data.
The `Encoder` and `Decoder` types are streaming tokenizers concerned with the
packing or parsing of JSON data. They operate on `Token` and `RawValue` types
which represent the common data structures that are representable in JSON.
`Encoder` and `Decoder` do not aim to provide any interpretation of the data.
Functions like `Marshal`, `MarshalFull`, `MarshalNext`, `Unmarshal`,
`UnmarshalFull`, and `UnmarshalNext` provide semantic meaning by correlating
any arbitrary Go type with some JSON representation of that type (as stored in
data types like `[]byte`, `io.Writer`, `io.Reader`, `Encoder`, or `Decoder`).
![API overview](api.png)
This diagram provides a high-level overview of the v2 `json` package.
Purple blocks represent types, while blue blocks represent functions or methods.
The arrows and their direction represent the approximate flow of data.
The bottom half of the diagram contains functionality that is only concerned
with syntax, while the upper half contains functionality that assigns
semantic meaning to syntactic data handled by the bottom half.
In contrast to v1 `encoding/json`, options are represented as separate types
rather than being setter methods on the `Encoder` or `Decoder` types.
## Behavior changes
The v2 `json` package changes the default behavior of `Marshal` and `Unmarshal`
relative to the v1 `json` package to be more sensible.
Some of these behavior changes have options and workarounds to opt into
behavior similar to what v1 provided.
This table shows an overview of the changes:
| v1 | v2 | Details |
| -- | -- | ------- |
| JSON object members are unmarshaled into a Go struct using a **case-insensitive name match**. | JSON object members are unmarshaled into a Go struct using a **case-sensitive name match**. | [CaseSensitivity](/diff_test.go#:~:text=TestCaseSensitivity) |
| When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value is an empty Go value**, which is defined as false, 0, a nil pointer, a nil interface value, and any empty array, slice, map, or string. | When marshaling a Go struct, a struct field marked as `omitempty` is omitted if **the field value would encode as an empty JSON value**, which is defined as a JSON null, or an empty JSON string, object, or array. | [OmitEmptyOption](/diff_test.go#:~:text=TestOmitEmptyOption) |
| The `string` option **does affect** Go bools. | The `string` option **does not affect** Go bools. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| The `string` option **does not recursively affect** sub-values of the Go field value. | The `string` option **does recursively affect** sub-values of the Go field value. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| The `string` option **sometimes accepts** a JSON null escaped within a JSON string. | The `string` option **never accepts** a JSON null escaped within a JSON string. | [StringOption](/diff_test.go#:~:text=TestStringOption) |
| A nil Go slice is marshaled as a **JSON null**. | A nil Go slice is marshaled as an **empty JSON array**. | [NilSlicesAndMaps](/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A nil Go map is marshaled as a **JSON null**. | A nil Go map is marshaled as an **empty JSON object**. | [NilSlicesAndMaps](/diff_test.go#:~:text=TestNilSlicesAndMaps) |
| A Go array may be unmarshaled from a **JSON array of any length**. | A Go array must be unmarshaled from a **JSON array of the same length**. | [Arrays](/diff_test.go#:~:text=Arrays) |
| A Go byte array is represented as a **JSON array of JSON numbers**. | A Go byte array is represented as a **Base64-encoded JSON string**. | [ByteArrays](/diff_test.go#:~:text=TestByteArrays) |
| `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **inconsistently called**. | `MarshalJSON` and `UnmarshalJSON` methods declared on a pointer receiver are **consistently called**. | [PointerReceiver](/diff_test.go#:~:text=TestPointerReceiver) |
| A Go map is marshaled in a **deterministic order**. | A Go map is marshaled in a **non-deterministic order**. | [MapDeterminism](/diff_test.go#:~:text=TestMapDeterminism) |
| JSON strings are encoded **with HTML-specific characters being escaped**. | JSON strings are encoded **without any characters being escaped** (unless necessary). | [EscapeHTML](/diff_test.go#:~:text=TestEscapeHTML) |
| When marshaling, invalid UTF-8 within a Go string **are silently replaced**. | When marshaling, invalid UTF-8 within a Go string **results in an error**. | [InvalidUTF8](/diff_test.go#:~:text=TestInvalidUTF8) |
| When unmarshaling, invalid UTF-8 within a JSON string **are silently replaced**. | When unmarshaling, invalid UTF-8 within a JSON string **results in an error**. | [InvalidUTF8](/diff_test.go#:~:text=TestInvalidUTF8) |
| When marshaling, **an error does not occur** if the output JSON value contains objects with duplicate names. | When marshaling, **an error does occur** if the output JSON value contains objects with duplicate names. | [DuplicateNames](/diff_test.go#:~:text=TestDuplicateNames) |
| When unmarshaling, **an error does not occur** if the input JSON value contains objects with duplicate names. | When unmarshaling, **an error does occur** if the input JSON value contains objects with duplicate names. | [DuplicateNames](/diff_test.go#:~:text=TestDuplicateNames) |
| Unmarshaling a JSON null into a non-empty Go value **inconsistently clears the value or does nothing**. | Unmarshaling a JSON null into a non-empty Go value **always clears the value**. | [MergeNull](/diff_test.go#:~:text=TestMergeNull) |
| Unmarshaling a JSON value into a non-empty Go value **follows inconsistent and bizarre behavior**. | Unmarshaling a JSON value into a non-empty Go value **always merges if the input is an object, and otherwise replaces**. | [MergeComposite](/diff_test.go#:~:text=TestMergeComposite) |
| A `time.Duration` is represented as a **JSON number containing the decimal number of nanoseconds**. | A `time.Duration` is represented as a **JSON string containing the formatted duration (e.g., "1h2m3.456s")**. | [TimeDurations](/diff_test.go#:~:text=TestTimeDurations) |
| Unmarshaling a JSON number into a Go float beyond its representation **results in an error**. | Unmarshaling a JSON number into a Go float beyond its representation **uses the closest representable value (e.g., ±`math.MaxFloat`)**. | [MaxFloats](/diff_test.go#:~:text=TestMaxFloats) |
| A Go struct with only unexported fields **can be serialized**. | A Go struct with only unexported fields **cannot be serialized**. | [EmptyStructs](/diff_test.go#:~:text=TestEmptyStructs) |
| A Go struct that embeds an unexported struct type **can sometimes be serialized**. | A Go struct that embeds an unexported struct type **cannot be serialized**. | [EmbedUnexported](/diff_test.go#:~:text=TestEmbedUnexported) |
See [diff_test.go](/diff_test.go) for details about every change.
## Performance
One of the goals of the v2 module is to be more performant than v1.
Each of the charts below show the performance across
several different JSON implementations:
* `JSONv1` is `encoding/json` at `v1.18.2`
* `JSONv2` is `github.com/go-json-experiment/json` at `v0.0.0-20220524042235-dd8be80fc4a7`
* `JSONIterator` is `github.com/json-iterator/go` at `v1.1.12`
* `SegmentJSON` is `github.com/segmentio/encoding/json` at `v0.3.5`
* `GoJSON` is `github.com/goccy/go-json` at `v0.9.7`
* `SonicJSON` is `github.com/bytedance/sonic` at `v1.3.0`
Benchmarks were run across various datasets:
* `CanadaGeometry` is a GeoJSON (RFC 7946) representation of Canada.
It contains many JSON arrays of arrays of two-element arrays of numbers.
* `CITMCatalog` contains many JSON objects using numeric names.
* `SyntheaFHIR` is sample JSON data from the healthcare industry.
It contains many nested JSON objects with mostly string values,
where the set of unique string values is relatively small.
* `TwitterStatus` is the JSON response from the Twitter API.
It contains a mix of all different JSON kinds, where string values
are a mix of both single-byte ASCII and multi-byte Unicode.
* `GolangSource` is a simple tree representing the Go source code.
It contains many nested JSON objects, each with the same schema.
* `StringUnicode` contains many strings with multi-byte Unicode runes.
All of the implementations other than `JSONv1` and `JSONv2` make
extensive use of `unsafe`. As such, we expect those to generally be faster,
but at the cost of memory and type safety. `SonicJSON` goes a step even further
and uses just-in-time compilation to generate machine code specialized
for the Go type being marshaled or unmarshaled.
Also, `SonicJSON` does not validate JSON strings for valid UTF-8,
and so gains a notable performance boost on datasets with multi-byte Unicode.
Benchmarks are performed based on the default marshal and unmarshal behavior
of each package. Note that `JSONv2` aims to be safe and correct by default,
which may not be the most performant strategy.
`JSONv2` has several semantic changes relative to `JSONv1` that
impacts performance:
1. When marshaling, `JSONv2` no longer sorts the keys of a Go map.
This will improve performance.
2. When marshaling or unmarshaling, `JSONv2` always checks
to make sure JSON object names are unique.
This will hurt performance, but is more correct.
3. When marshaling or unmarshaling, `JSONv2` always
shallow copies the underlying value for a Go interface and
shallow copies the key and value for entries in a Go map.
This is done to keep the value as addressable so that `JSONv2` can
call methods and functions that operate on a pointer receiver.
This will hurt performance, but is more correct.
All of the charts are unit-less since the values are normalized
relative to `JSONv1`, which is why `JSONv1` always has a value of 1.
A lower value is better (i.e., runs faster).
Benchmarks were performed on an AMD Ryzen 9 5900X.
The code for the benchmarks is located at
https://github.com/go-json-experiment/jsonbench.
### Marshal Performance
#### Concrete types
![Benchmark Marshal Concrete](benchmark-marshal-concrete.png)
* This compares marshal performance when serializing
[from concrete types](/testdata_test.go).
* The `JSONv1` implementation is close to optimal (without the use of `unsafe`).
* Relative to `JSONv1`, `JSONv2` is generally as fast or slightly faster.
* Relative to `JSONIterator`, `JSONv2` is up to 1.3x faster.
* Relative to `SegmentJSON`, `JSONv2` is up to 1.8x slower.
* Relative to `GoJSON`, `JSONv2` is up to 2.0x slower.
* Relative to `SonicJSON`, `JSONv2` is about 1.8x to 3.2x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* For `JSONv1` and `JSONv2`, marshaling from concrete types is
mostly limited by the performance of Go reflection.
#### Interface types
![Benchmark Marshal Interface](benchmark-marshal-interface.png)
* This compares marshal performance when serializing from
`any`, `map[string]any`, and `[]any` types.
* Relative to `JSONv1`, `JSONv2` is about 1.5x to 4.2x faster.
* Relative to `JSONIterator`, `JSONv2` is about 1.1x to 2.4x faster.
* Relative to `SegmentJSON`, `JSONv2` is about 1.2x to 1.8x faster.
* Relative to `GoJSON`, `JSONv2` is about 1.1x to 2.5x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* `JSONv2` is faster than the alternatives.
One advantange is because it does not sort the keys for a `map[string]any`,
while alternatives (except `SonicJSON` and `JSONIterator`) do sort the keys.
#### RawValue types
![Benchmark Marshal Rawvalue](benchmark-marshal-rawvalue.png)
* This compares performance when marshaling from a `json.RawValue`.
This mostly exercises the underlying encoder and
hides the cost of Go reflection.
* Relative to `JSONv1`, `JSONv2` is about 3.5x to 7.8x faster.
* `JSONIterator` is blazingly fast because
[it does not validate whether the raw value is valid](https://go.dev/play/p/bun9IXQCKRe)
and simply copies it to the output.
* Relative to `SegmentJSON`, `JSONv2` is about 1.5x to 2.7x faster.
* Relative to `GoJSON`, `JSONv2` is up to 2.2x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x faster.
* Aside from `JSONIterator`, `JSONv2` is generally the fastest.
### Unmarshal Performance
#### Concrete types
![Benchmark Unmarshal Concrete](benchmark-unmarshal-concrete.png)
* This compares unmarshal performance when deserializing
[into concrete types](/testdata_test.go).
* Relative to `JSONv1`, `JSONv2` is about 1.8x to 5.7x faster.
* Relative to `JSONIterator`, `JSONv2` is about 1.1x to 1.6x slower.
* Relative to `SegmentJSON`, `JSONv2` is up to 2.5x slower.
* Relative to `GoJSON`, `JSONv2` is about 1.4x to 2.1x slower.
* Relative to `SonicJSON`, `JSONv2` is up to 4.0x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* For `JSONv1` and `JSONv2`, unmarshaling into concrete types is
mostly limited by the performance of Go reflection.
#### Interface types
![Benchmark Unmarshal Interface](benchmark-unmarshal-interface.png)
* This compares unmarshal performance when deserializing into
`any`, `map[string]any`, and `[]any` types.
* Relative to `JSONv1`, `JSONv2` is about 1.tx to 4.3x faster.
* Relative to `JSONIterator`, `JSONv2` is up to 1.5x faster.
* Relative to `SegmentJSON`, `JSONv2` is about 1.5 to 3.7x faster.
* Relative to `GoJSON`, `JSONv2` is up to 1.3x faster.
* Relative to `SonicJSON`, `JSONv2` is up to 1.5x slower
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* Aside from `SonicJSON`, `JSONv2` is generally just as fast
or faster than all the alternatives.
#### RawValue types
![Benchmark Unmarshal Rawvalue](benchmark-unmarshal-rawvalue.png)
* This compares performance when unmarshaling into a `json.RawValue`.
This mostly exercises the underlying decoder and
hides away most of the cost of Go reflection.
* Relative to `JSONv1`, `JSONv2` is about 8.3x to 17.0x faster.
* Relative to `JSONIterator`, `JSONv2` is up to 2.0x faster.
* Relative to `SegmentJSON`, `JSONv2` is up to 1.6x faster or 1.7x slower.
* Relative to `GoJSON`, `JSONv2` is up to 1.9x faster or 2.1x slower.
* Relative to `SonicJSON`, `JSONv2` is up to 2.0x faster
(ignoring `StringUnicode` since `SonicJSON` does not validate UTF-8).
* `JSONv1` takes a
[lexical scanning approach](https://talks.golang.org/2011/lex.slide#1),
which performs a virtual function call for every byte of input.
In contrast, `JSONv2` makes heavy use of iterative and linear parsing logic
(with extra complexity to resume parsing when encountering segmented buffers).
* `JSONv2` is comparable to the alternatives that use `unsafe`.
Generally it is faster, but sometimes it is slower.

View File

@ -0,0 +1,506 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"io"
"reflect"
"sync"
)
// MarshalOptions configures how Go data is serialized as JSON data.
// The zero value is equivalent to the default marshal settings.
type MarshalOptions struct {
requireKeyedLiterals
nonComparable
// Marshalers is a list of type-specific marshalers to use.
Marshalers *Marshalers
// StringifyNumbers specifies that numeric Go types should be serialized
// as a JSON string containing the equivalent JSON number value.
//
// According to RFC 8259, section 6, a JSON implementation may choose to
// limit the representation of a JSON number to an IEEE 754 binary64 value.
// This may cause decoders to lose precision for int64 and uint64 types.
// Escaping JSON numbers as a JSON string preserves the exact precision.
StringifyNumbers bool
// DiscardUnknownMembers specifies that marshaling should ignore any
// JSON object members stored in Go struct fields dedicated to storing
// unknown JSON object members.
DiscardUnknownMembers bool
// formatDepth is the depth at which we respect the format flag.
formatDepth int
// format is custom formatting for the value at the specified depth.
format string
}
// Marshal serializes a Go value as a []byte with default options.
// It is a thin wrapper over MarshalOptions.Marshal.
func Marshal(in any) (out []byte, err error) {
return MarshalOptions{}.Marshal(EncodeOptions{}, in)
}
// MarshalFull serializes a Go value into an io.Writer with default options.
// It is a thin wrapper over MarshalOptions.MarshalFull.
func MarshalFull(out io.Writer, in any) error {
return MarshalOptions{}.MarshalFull(EncodeOptions{}, out, in)
}
// Marshal serializes a Go value as a []byte according to the provided
// marshal and encode options. It does not terminate the output with a newline.
// See MarshalNext for details about the conversion of a Go value into JSON.
func (mo MarshalOptions) Marshal(eo EncodeOptions, in any) (out []byte, err error) {
enc := getBufferedEncoder(eo)
defer putBufferedEncoder(enc)
enc.options.omitTopLevelNewline = true
err = mo.MarshalNext(enc, in)
// TODO(https://go.dev/issue/45038): Use bytes.Clone.
return append([]byte(nil), enc.buf...), err
}
// MarshalFull serializes a Go value into an io.Writer according to the provided
// marshal and encode options. It does not terminate the output with a newline.
// See MarshalNext for details about the conversion of a Go value into JSON.
func (mo MarshalOptions) MarshalFull(eo EncodeOptions, out io.Writer, in any) error {
enc := getStreamingEncoder(out, eo)
defer putStreamingEncoder(enc)
enc.options.omitTopLevelNewline = true
err := mo.MarshalNext(enc, in)
return err
}
// MarshalNext encodes a Go value as the next JSON value according to
// the provided marshal options.
//
// Type-specific marshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when encoding
// a value of type T (by taking its address) or a non-nil value of *T.
// MarshalNext ensures that a value is always addressable
// (by boxing it on the heap if necessary) so that
// these functions and methods can be consistently called. For performance,
// it is recommended that MarshalNext be passed a non-nil pointer to the value.
//
// The input value is encoded as JSON according the following rules:
//
// - If any type-specific functions in MarshalOptions.Marshalers match
// the value type, then those functions are called to encode the value.
// If all applicable functions return SkipFunc,
// then the value is encoded according to subsequent rules.
//
// - If the value type implements MarshalerV2,
// then the MarshalNextJSON method is called to encode the value.
//
// - If the value type implements MarshalerV1,
// then the MarshalJSON method is called to encode the value.
//
// - If the value type implements encoding.TextMarshaler,
// then the MarshalText method is called to encode the value and
// subsequently encode its result as a JSON string.
//
// - Otherwise, the value is encoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// Certain types support specialized formatting according to
// a format flag optionally specified in the Go struct tag
// for the struct field that contains the current value
// (see the “JSON Representation of Go structs” section for more details).
//
// The representation of each type is as follows:
//
// - A Go boolean is encoded as a JSON boolean (e.g., true or false).
// It does not support any custom format flags.
//
// - A Go string is encoded as a JSON string.
// It does not support any custom format flags.
//
// - A Go []byte or [N]byte is encoded as a JSON string containing
// the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the bytes value is encoded as a JSON array
// where each byte is recursively JSON-encoded as each JSON array element.
//
// - A Go integer is encoded as a JSON number without fractions or exponents.
// If MarshalOptions.StringifyNumbers is specified, then the JSON number is
// encoded within a JSON string. It does not support any custom format
// flags.
//
// - A Go float is encoded as a JSON number.
// If MarshalOptions.StringifyNumbers is specified,
// then the JSON number is encoded within a JSON string.
// If the format is "nonfinite", then NaN, +Inf, and -Inf are encoded as
// the JSON strings "NaN", "Infinity", and "-Infinity", respectively.
// Otherwise, the presence of non-finite numbers results in a SemanticError.
//
// - A Go map is encoded as a JSON object, where each Go map key and value
// is recursively encoded as a name and value pair in the JSON object.
// The Go map key must encode as a JSON string, otherwise this results
// in a SemanticError. When encoding keys, MarshalOptions.StringifyNumbers
// is automatically applied so that numeric keys encode as JSON strings.
// The Go map is traversed in a non-deterministic order.
// For deterministic encoding, consider using RawValue.Canonicalize.
// If the format is "emitnull", then a nil map is encoded as a JSON null.
// Otherwise by default, a nil map is encoded as an empty JSON object.
//
// - A Go struct is encoded as a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is encoded as a JSON array, where each Go slice element
// is recursively JSON-encoded as the elements of the JSON array.
// If the format is "emitnull", then a nil slice is encoded as a JSON null.
// Otherwise by default, a nil slice is encoded as an empty JSON array.
//
// - A Go array is encoded as a JSON array, where each Go array element
// is recursively JSON-encoded as the elements of the JSON array.
// The JSON array length is always identical to the Go array length.
// It does not support any custom format flags.
//
// - A Go pointer is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
// Format flags are forwarded to the encoding of the underlying value.
//
// - A Go interface is encoded as a JSON null if nil, otherwise it is
// the recursively JSON-encoded representation of the underlying value.
// It does not support any custom format flags.
//
// - A Go time.Time is encoded as a JSON string containing the timestamp
// formatted in RFC 3339 with nanosecond resolution.
// If the format matches one of the format constants declared
// in the time package (e.g., RFC1123), then that format is used.
// Otherwise, the format is used as-is with time.Time.Format if non-empty.
//
// - A Go time.Duration is encoded as a JSON string containing the duration
// formatted according to time.Duration.String.
// If the format is "nanos", it is encoded as a JSON number
// containing the number of nanoseconds in the duration.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a SemanticError.
//
// JSON cannot represent cyclic data structures and
// MarshalNext does not handle them.
// Passing cyclic structures will result in an error.
func (mo MarshalOptions) MarshalNext(out *Encoder, in any) error {
v := reflect.ValueOf(in)
if !v.IsValid() || (v.Kind() == reflect.Pointer && v.IsNil()) {
return out.WriteToken(Null)
}
// Shallow copy non-pointer values to obtain an addressable value.
// It is beneficial to performance to always pass pointers to avoid this.
if v.Kind() != reflect.Pointer {
v2 := reflect.New(v.Type())
v2.Elem().Set(v)
v = v2
}
va := addressableValue{v.Elem()} // dereferenced pointer is always addressable
t := va.Type()
// Lookup and call the marshal function for this type.
marshal := lookupArshaler(t).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, t)
}
if err := marshal(mo, out, va); err != nil {
if !out.options.AllowDuplicateNames {
out.tokens.invalidateDisabledNamespaces()
}
return err
}
return nil
}
// UnmarshalOptions configures how JSON data is deserialized as Go data.
// The zero value is equivalent to the default unmarshal settings.
type UnmarshalOptions struct {
requireKeyedLiterals
nonComparable
// Unmarshalers is a list of type-specific unmarshalers to use.
Unmarshalers *Unmarshalers
// StringifyNumbers specifies that numeric Go types can be deserialized
// from either a JSON number or a JSON string containing a JSON number
// without any surrounding whitespace.
StringifyNumbers bool
// RejectUnknownMembers specifies that unknown members should be rejected
// when unmarshaling a JSON object, regardless of whether there is a field
// to store unknown members.
RejectUnknownMembers bool
// formatDepth is the depth at which we respect the format flag.
formatDepth int
// format is custom formatting for the value at the specified depth.
format string
}
// Unmarshal deserializes a Go value from a []byte with default options.
// It is a thin wrapper over UnmarshalOptions.Unmarshal.
func Unmarshal(in []byte, out any) error {
return UnmarshalOptions{}.Unmarshal(DecodeOptions{}, in, out)
}
// UnmarshalFull deserializes a Go value from an io.Reader with default options.
// It is a thin wrapper over UnmarshalOptions.UnmarshalFull.
func UnmarshalFull(in io.Reader, out any) error {
return UnmarshalOptions{}.UnmarshalFull(DecodeOptions{}, in, out)
}
// Unmarshal deserializes a Go value from a []byte according to the
// provided unmarshal and decode options. The output must be a non-nil pointer.
// The input must be a single JSON value with optional whitespace interspersed.
// See UnmarshalNext for details about the conversion of JSON into a Go value.
func (uo UnmarshalOptions) Unmarshal(do DecodeOptions, in []byte, out any) error {
dec := getBufferedDecoder(in, do)
defer putBufferedDecoder(dec)
return uo.unmarshalFull(dec, out)
}
// UnmarshalFull deserializes a Go value from an io.Reader according to the
// provided unmarshal and decode options. The output must be a non-nil pointer.
// The input must be a single JSON value with optional whitespace interspersed.
// It consumes the entirety of io.Reader until io.EOF is encountered.
// See UnmarshalNext for details about the conversion of JSON into a Go value.
func (uo UnmarshalOptions) UnmarshalFull(do DecodeOptions, in io.Reader, out any) error {
dec := getStreamingDecoder(in, do)
defer putStreamingDecoder(dec)
return uo.unmarshalFull(dec, out)
}
func (uo UnmarshalOptions) unmarshalFull(in *Decoder, out any) error {
switch err := uo.UnmarshalNext(in, out); err {
case nil:
return in.checkEOF()
case io.EOF:
return io.ErrUnexpectedEOF
default:
return err
}
}
// UnmarshalNext decodes the next JSON value into a Go value according to
// the provided unmarshal options. The output must be a non-nil pointer.
//
// Type-specific unmarshal functions and methods take precedence
// over the default representation of a value.
// Functions or methods that operate on *T are only called when decoding
// a value of type T (by taking its address) or a non-nil value of *T.
// UnmarshalNext ensures that a value is always addressable
// (by boxing it on the heap if necessary) so that
// these functions and methods can be consistently called.
//
// The input is decoded into the output according the following rules:
//
// - If any type-specific functions in UnmarshalOptions.Unmarshalers match
// the value type, then those functions are called to decode the JSON
// value. If all applicable functions return SkipFunc,
// then the input is decoded according to subsequent rules.
//
// - If the value type implements UnmarshalerV2,
// then the UnmarshalNextJSON method is called to decode the JSON value.
//
// - If the value type implements UnmarshalerV1,
// then the UnmarshalJSON method is called to decode the JSON value.
//
// - If the value type implements encoding.TextUnmarshaler,
// then the input is decoded as a JSON string and
// the UnmarshalText method is called with the decoded string value.
// This fails with a SemanticError if the input is not a JSON string.
//
// - Otherwise, the JSON value is decoded according to the value's type
// as described in detail below.
//
// Most Go types have a default JSON representation.
// Certain types support specialized formatting according to
// a format flag optionally specified in the Go struct tag
// for the struct field that contains the current value
// (see the “JSON Representation of Go structs” section for more details).
// A JSON null may be decoded into every supported Go value where
// it is equivalent to storing the zero value of the Go value.
// If the input JSON kind is not handled by the current Go value type,
// then this fails with a SemanticError. Unless otherwise specified,
// the decoded value replaces any pre-existing value.
//
// The representation of each type is as follows:
//
// - A Go boolean is decoded from a JSON boolean (e.g., true or false).
// It does not support any custom format flags.
//
// - A Go string is decoded from a JSON string.
// It does not support any custom format flags.
//
// - A Go []byte or [N]byte is decoded from a JSON string
// containing the binary value encoded using RFC 4648.
// If the format is "base64" or unspecified, then this uses RFC 4648, section 4.
// If the format is "base64url", then this uses RFC 4648, section 5.
// If the format is "base32", then this uses RFC 4648, section 6.
// If the format is "base32hex", then this uses RFC 4648, section 7.
// If the format is "base16" or "hex", then this uses RFC 4648, section 8.
// If the format is "array", then the Go slice or array is decoded from a
// JSON array where each JSON element is recursively decoded for each byte.
// When decoding into a non-nil []byte, the slice length is reset to zero
// and the decoded input is appended to it.
// When decoding into a [N]byte, the input must decode to exactly N bytes,
// otherwise it fails with a SemanticError.
//
// - A Go integer is decoded from a JSON number.
// It may also be decoded from a JSON string containing a JSON number
// if UnmarshalOptions.StringifyNumbers is specified.
// It fails with a SemanticError if the JSON number
// has a fractional or exponent component.
// It also fails if it overflows the representation of the Go integer type.
// It does not support any custom format flags.
//
// - A Go float is decoded from a JSON number.
// It may also be decoded from a JSON string containing a JSON number
// if UnmarshalOptions.StringifyNumbers is specified.
// The JSON number is parsed as the closest representable Go float value.
// If the format is "nonfinite", then the JSON strings
// "NaN", "Infinity", and "-Infinity" are decoded as NaN, +Inf, and -Inf.
// Otherwise, the presence of such strings results in a SemanticError.
//
// - A Go map is decoded from a JSON object,
// where each JSON object name and value pair is recursively decoded
// as the Go map key and value. When decoding keys,
// UnmarshalOptions.StringifyNumbers is automatically applied so that
// numeric keys can decode from JSON strings. Maps are not cleared.
// If the Go map is nil, then a new map is allocated to decode into.
// If the decoded key matches an existing Go map entry, the entry value
// is reused by decoding the JSON object value into it.
// The only supported format is "emitnull" and has no effect when decoding.
//
// - A Go struct is decoded from a JSON object.
// See the “JSON Representation of Go structs” section
// in the package-level documentation for more details.
//
// - A Go slice is decoded from a JSON array, where each JSON element
// is recursively decoded and appended to the Go slice.
// Before appending into a Go slice, a new slice is allocated if it is nil,
// otherwise the slice length is reset to zero.
// The only supported format is "emitnull" and has no effect when decoding.
//
// - A Go array is decoded from a JSON array, where each JSON array element
// is recursively decoded as each corresponding Go array element.
// Each Go array element is zeroed before decoding into it.
// It fails with a SemanticError if the JSON array does not contain
// the exact same number of elements as the Go array.
// It does not support any custom format flags.
//
// - A Go pointer is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil pointer.
// Otherwise, it allocates a new underlying value if the pointer is nil,
// and recursively JSON decodes into the underlying value.
// Format flags are forwarded to the decoding of the underlying type.
//
// - A Go interface is decoded based on the JSON kind and underlying Go type.
// If the input is a JSON null, then this stores a nil interface value.
// Otherwise, a nil interface value of an empty interface type is initialized
// with a zero Go bool, string, float64, map[string]any, or []any if the
// input is a JSON boolean, string, number, object, or array, respectively.
// If the interface value is still nil, then this fails with a SemanticError
// since decoding could not determine an appropriate Go type to decode into.
// For example, unmarshaling into a nil io.Reader fails since
// there is no concrete type to populate the interface value with.
// Otherwise an underlying value exists and it recursively decodes
// the JSON input into it. It does not support any custom format flags.
//
// - A Go time.Time is decoded from a JSON string containing the time
// formatted in RFC 3339 with nanosecond resolution.
// If the format matches one of the format constants declared in
// the time package (e.g., RFC1123), then that format is used for parsing.
// Otherwise, the format is used as-is with time.Time.Parse if non-empty.
//
// - A Go time.Duration is decoded from a JSON string by
// passing the decoded string to time.ParseDuration.
// If the format is "nanos", it is instead decoded from a JSON number
// containing the number of nanoseconds in the duration.
//
// - All other Go types (e.g., complex numbers, channels, and functions)
// have no default representation and result in a SemanticError.
//
// In general, unmarshaling follows merge semantics (similar to RFC 7396)
// where the decoded Go value replaces the destination value
// for any JSON kind other than an object.
// For JSON objects, the input object is merged into the destination value
// where matching object members recursively apply merge semantics.
func (uo UnmarshalOptions) UnmarshalNext(in *Decoder, out any) error {
v := reflect.ValueOf(out)
if !v.IsValid() || v.Kind() != reflect.Pointer || v.IsNil() {
var t reflect.Type
if v.IsValid() {
t = v.Type()
if t.Kind() == reflect.Pointer {
t = t.Elem()
}
}
err := errors.New("value must be passed as a non-nil pointer reference")
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
va := addressableValue{v.Elem()} // dereferenced pointer is always addressable
t := va.Type()
// Lookup and call the unmarshal function for this type.
unmarshal := lookupArshaler(t).unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.lookup(unmarshal, t)
}
if err := unmarshal(uo, in, va); err != nil {
if !in.options.AllowDuplicateNames {
in.tokens.invalidateDisabledNamespaces()
}
return err
}
return nil
}
// addressableValue is a reflect.Value that is guaranteed to be addressable
// such that calling the Addr and Set methods do not panic.
//
// There is no compile magic that enforces this property,
// but rather the need to construct this type makes it easier to examine each
// construction site to ensure that this property is upheld.
type addressableValue struct{ reflect.Value }
// newAddressableValue constructs a new addressable value of type t.
func newAddressableValue(t reflect.Type) addressableValue {
return addressableValue{reflect.New(t).Elem()}
}
// All marshal and unmarshal behavior is implemented using these signatures.
type (
marshaler = func(MarshalOptions, *Encoder, addressableValue) error
unmarshaler = func(UnmarshalOptions, *Decoder, addressableValue) error
)
type arshaler struct {
marshal marshaler
unmarshal unmarshaler
nonDefault bool
}
var lookupArshalerCache sync.Map // map[reflect.Type]*arshaler
func lookupArshaler(t reflect.Type) *arshaler {
if v, ok := lookupArshalerCache.Load(t); ok {
return v.(*arshaler)
}
fncs := makeDefaultArshaler(t)
fncs = makeMethodArshaler(fncs, t)
fncs = makeTimeArshaler(fncs, t)
// Use the last stored so that duplicate arshalers can be garbage collected.
v, _ := lookupArshalerCache.LoadOrStore(t, fncs)
return v.(*arshaler)
}

View File

@ -0,0 +1,219 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import "reflect"
// This files contains an optimized marshal and unmarshal implementation
// for the any type. This type is often used when the Go program has
// no knowledge of the JSON schema. This is a common enough occurrence
// to justify the complexity of adding logic for this.
func marshalValueAny(mo MarshalOptions, enc *Encoder, val any) error {
switch val := val.(type) {
case nil:
return enc.WriteToken(Null)
case bool:
return enc.WriteToken(Bool(val))
case string:
return enc.WriteToken(String(val))
case float64:
return enc.WriteToken(Float(val))
case map[string]any:
return marshalObjectAny(mo, enc, val)
case []any:
return marshalArrayAny(mo, enc, val)
default:
v := newAddressableValue(reflect.TypeOf(val))
v.Set(reflect.ValueOf(val))
marshal := lookupArshaler(v.Type()).marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, v.Type())
}
return marshal(mo, enc, v)
}
}
func unmarshalValueAny(uo UnmarshalOptions, dec *Decoder) (any, error) {
switch k := dec.PeekKind(); k {
case '{':
return unmarshalObjectAny(uo, dec)
case '[':
return unmarshalArrayAny(uo, dec)
default:
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return nil, err
}
switch val.Kind() {
case 'n':
return nil, nil
case 'f':
return false, nil
case 't':
return true, nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
if dec.stringCache == nil {
dec.stringCache = new(stringCache)
}
return dec.stringCache.make(val), nil
case '0':
fv, _ := parseFloat(val, 64) // ignore error since readValue gaurantees val is valid
return fv, nil
default:
panic("BUG: invalid kind: " + k.String())
}
}
}
func marshalObjectAny(mo MarshalOptions, enc *Encoder, obj map[string]any) error {
// Check for cycles.
if enc.tokens.depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(obj)
if err := enc.seenPointers.visit(v); err != nil {
return err
}
defer enc.seenPointers.leave(v)
}
// Optimize for marshaling an empty map without any preceding whitespace.
if len(obj) == 0 && !enc.options.multiline && !enc.tokens.last.needObjectName() {
enc.buf = enc.tokens.mayAppendDelim(enc.buf, '{')
enc.buf = append(enc.buf, "{}"...)
enc.tokens.last.increment()
if enc.needFlush() {
return enc.flush()
}
return nil
}
if err := enc.WriteToken(ObjectStart); err != nil {
return err
}
// A Go map guarantees that each entry has a unique key
// The only possibility of duplicates is due to invalid UTF-8.
if !enc.options.AllowInvalidUTF8 {
enc.tokens.last.disableNamespace()
}
for name, val := range obj {
if err := enc.WriteToken(String(name)); err != nil {
return err
}
if err := marshalValueAny(mo, enc, val); err != nil {
return err
}
}
if err := enc.WriteToken(ObjectEnd); err != nil {
return err
}
return nil
}
func unmarshalObjectAny(uo UnmarshalOptions, dec *Decoder) (map[string]any, error) {
tok, err := dec.ReadToken()
if err != nil {
return nil, err
}
k := tok.Kind()
switch k {
case 'n':
return nil, nil
case '{':
obj := make(map[string]any)
// A Go map guarantees that each entry has a unique key
// The only possibility of duplicates is due to invalid UTF-8.
if !dec.options.AllowInvalidUTF8 {
dec.tokens.last.disableNamespace()
}
for dec.PeekKind() != '}' {
tok, err := dec.ReadToken()
if err != nil {
return obj, err
}
name := tok.String()
// Manually check for duplicate names.
if _, ok := obj[name]; ok {
name := dec.previousBuffer()
err := &SyntacticError{str: "duplicate name " + string(name) + " in object"}
return obj, err.withOffset(dec.InputOffset() - int64(len(name)))
}
val, err := unmarshalValueAny(uo, dec)
obj[name] = val
if err != nil {
return obj, err
}
}
if _, err := dec.ReadToken(); err != nil {
return obj, err
}
return obj, nil
}
return nil, &SemanticError{action: "unmarshal", JSONKind: k, GoType: mapStringAnyType}
}
func marshalArrayAny(mo MarshalOptions, enc *Encoder, arr []any) error {
// Check for cycles.
if enc.tokens.depth() > startDetectingCyclesAfter {
v := reflect.ValueOf(arr)
if err := enc.seenPointers.visit(v); err != nil {
return err
}
defer enc.seenPointers.leave(v)
}
// Optimize for marshaling an empty slice without any preceding whitespace.
if len(arr) == 0 && !enc.options.multiline && !enc.tokens.last.needObjectName() {
enc.buf = enc.tokens.mayAppendDelim(enc.buf, '[')
enc.buf = append(enc.buf, "[]"...)
enc.tokens.last.increment()
if enc.needFlush() {
return enc.flush()
}
return nil
}
if err := enc.WriteToken(ArrayStart); err != nil {
return err
}
for _, val := range arr {
if err := marshalValueAny(mo, enc, val); err != nil {
return err
}
}
if err := enc.WriteToken(ArrayEnd); err != nil {
return err
}
return nil
}
func unmarshalArrayAny(uo UnmarshalOptions, dec *Decoder) ([]any, error) {
tok, err := dec.ReadToken()
if err != nil {
return nil, err
}
k := tok.Kind()
switch k {
case 'n':
return nil, nil
case '[':
arr := []any{}
for dec.PeekKind() != ']' {
val, err := unmarshalValueAny(uo, dec)
arr = append(arr, val)
if err != nil {
return arr, err
}
}
if _, err := dec.ReadToken(); err != nil {
return arr, err
}
return arr, nil
}
return nil, &SemanticError{action: "unmarshal", JSONKind: k, GoType: sliceAnyType}
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,387 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"fmt"
"reflect"
"sync"
)
// SkipFunc may be returned by MarshalFuncV2 and UnmarshalFuncV2 functions.
//
// Any function that returns SkipFunc must not cause observable side effects
// on the provided Encoder or Decoder. For example, it is permissible to call
// Decoder.PeekKind, but not permissible to call Decoder.ReadToken or
// Encoder.WriteToken since such methods mutate the state.
const SkipFunc = jsonError("skip function")
// Marshalers is a list of functions that may override the marshal behavior
// of specific types. Populate MarshalOptions.Marshalers to use it.
// A nil *Marshalers is equivalent to an empty list.
type Marshalers = typedMarshalers
// NewMarshalers constructs a flattened list of marshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns SkipFunc, then the next applicable function is called,
// otherwise the default marshaling behavior is used.
//
// For example:
//
// m1 := NewMarshalers(f1, f2)
// m2 := NewMarshalers(f0, m1, f3) // equivalent to m3
// m3 := NewMarshalers(f0, f1, f2, f3) // equivalent to m2
func NewMarshalers(ms ...*Marshalers) *Marshalers {
return newMarshalers(ms...)
}
// Unmarshalers is a list of functions that may override the unmarshal behavior
// of specific types. Populate UnmarshalOptions.Unmarshalers to use it.
// A nil *Unmarshalers is equivalent to an empty list.
type Unmarshalers = typedUnmarshalers
// NewUnmarshalers constructs a flattened list of unmarshal functions.
// If multiple functions in the list are applicable for a value of a given type,
// then those earlier in the list take precedence over those that come later.
// If a function returns SkipFunc, then the next applicable function is called,
// otherwise the default unmarshaling behavior is used.
//
// For example:
//
// u1 := NewUnmarshalers(f1, f2)
// u2 := NewUnmarshalers(f0, u1, f3) // equivalent to u3
// u3 := NewUnmarshalers(f0, f1, f2, f3) // equivalent to u2
func NewUnmarshalers(us ...*Unmarshalers) *Unmarshalers {
return newUnmarshalers(us...)
}
type typedMarshalers = typedArshalers[MarshalOptions, Encoder]
type typedUnmarshalers = typedArshalers[UnmarshalOptions, Decoder]
type typedArshalers[Options, Coder any] struct {
nonComparable
fncVals []typedArshaler[Options, Coder]
fncCache sync.Map // map[reflect.Type]arshaler
// fromAny reports whether any of Go types used to represent arbitrary JSON
// (i.e., any, bool, string, float64, map[string]any, or []any) matches
// any of the provided type-specific arshalers.
//
// This bit of information is needed in arshal_default.go to determine
// whether to use the specialized logic in arshal_any.go to handle
// the any interface type. The logic in arshal_any.go does not support
// type-specific arshal functions, so we must avoid using that logic
// if this is true.
fromAny bool
}
type typedMarshaler = typedArshaler[MarshalOptions, Encoder]
type typedUnmarshaler = typedArshaler[UnmarshalOptions, Decoder]
type typedArshaler[Options, Coder any] struct {
typ reflect.Type
fnc func(Options, *Coder, addressableValue) error
maySkip bool
}
func newMarshalers(ms ...*Marshalers) *Marshalers { return newTypedArshalers(ms...) }
func newUnmarshalers(us ...*Unmarshalers) *Unmarshalers { return newTypedArshalers(us...) }
func newTypedArshalers[Options, Coder any](as ...*typedArshalers[Options, Coder]) *typedArshalers[Options, Coder] {
var a typedArshalers[Options, Coder]
for _, a2 := range as {
if a2 != nil {
a.fncVals = append(a.fncVals, a2.fncVals...)
a.fromAny = a.fromAny || a2.fromAny
}
}
if len(a.fncVals) == 0 {
return nil
}
return &a
}
func (a *typedArshalers[Options, Coder]) lookup(fnc func(Options, *Coder, addressableValue) error, t reflect.Type) (func(Options, *Coder, addressableValue) error, bool) {
if a == nil {
return fnc, false
}
if v, ok := a.fncCache.Load(t); ok {
if v == nil {
return fnc, false
}
return v.(func(Options, *Coder, addressableValue) error), true
}
// Collect a list of arshalers that can be called for this type.
// This list may be longer than 1 since some arshalers can be skipped.
var fncs []func(Options, *Coder, addressableValue) error
for _, fncVal := range a.fncVals {
if !castableTo(t, fncVal.typ) {
continue
}
fncs = append(fncs, fncVal.fnc)
if !fncVal.maySkip {
break // subsequent arshalers will never be called
}
}
if len(fncs) == 0 {
a.fncCache.Store(t, nil) // nil to indicate that no funcs found
return fnc, false
}
// Construct an arshaler that may call every applicable arshaler.
fncDefault := fnc
fnc = func(o Options, c *Coder, v addressableValue) error {
for _, fnc := range fncs {
if err := fnc(o, c, v); err != SkipFunc {
return err // may be nil or non-nil
}
}
return fncDefault(o, c, v)
}
// Use the first stored so duplicate work can be garbage collected.
v, _ := a.fncCache.LoadOrStore(t, fnc)
return v.(func(Options, *Coder, addressableValue) error), true
}
// MarshalFuncV1 constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value.
// The value of T must not be retained outside the function call.
// It may not return SkipFunc.
func MarshalFuncV1[T any](fn func(T) ([]byte, error)) *Marshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
val, err := fn(va.castTo(t).Interface().(T))
if err != nil {
err = wrapSkipFunc(err, "marshal function of type func(T) ([]byte, error)")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: RawValue(val).Kind(), GoType: t, Err: err}
}
return nil
},
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// MarshalFuncV2 constructs a type-specific marshaler that
// specifies how to marshal values of type T.
// T can be any type except a named pointer.
// The function is always provided with a non-nil pointer value
// if T is an interface or pointer type.
//
// The function must marshal exactly one JSON value by calling write methods
// on the provided encoder. It may return SkipFunc such that marshaling can
// move on to the next marshal function. However, no mutable method calls may
// be called on the encoder if SkipFunc is returned.
// The pointer to Encoder and the value of T must not be retained
// outside the function call.
func MarshalFuncV2[T any](fn func(MarshalOptions, *Encoder, T) error) *Marshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, true)
typFnc := typedMarshaler{
typ: t,
fnc: func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
prevDepth, prevLength := enc.tokens.depthLength()
err := fn(mo, enc, va.castTo(t).Interface().(T))
currDepth, currLength := enc.tokens.depthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errors.New("must write exactly one JSON value")
}
if err != nil {
if err == SkipFunc {
if prevDepth == currDepth && prevLength == currLength {
return SkipFunc
}
err = errors.New("must not write any JSON tokens when skipping")
}
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
return nil
},
maySkip: true,
}
return &Marshalers{fncVals: []typedMarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFuncV1 constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value.
// The input []byte must not be mutated.
// The input []byte and value T must not be retained outside the function call.
// It may not return SkipFunc.
func UnmarshalFuncV1[T any](fn func([]byte, T) error) *Unmarshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
err = fn(val, va.castTo(t).Interface().(T))
if err != nil {
err = wrapSkipFunc(err, "unmarshal function of type func([]byte, T) error")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
},
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// UnmarshalFuncV2 constructs a type-specific unmarshaler that
// specifies how to unmarshal values of type T.
// T must be an unnamed pointer or an interface type.
// The function is always provided with a non-nil pointer value.
//
// The function must unmarshal exactly one JSON value by calling read methods
// on the provided decoder. It may return SkipFunc such that unmarshaling can
// move on to the next unmarshal function. However, no mutable method calls may
// be called on the decoder if SkipFunc is returned.
// The pointer to Decoder and the value of T must not be retained
// outside the function call.
func UnmarshalFuncV2[T any](fn func(UnmarshalOptions, *Decoder, T) error) *Unmarshalers {
t := reflect.TypeOf((*T)(nil)).Elem()
assertCastableTo(t, false)
typFnc := typedUnmarshaler{
typ: t,
fnc: func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
prevDepth, prevLength := dec.tokens.depthLength()
err := fn(uo, dec, va.castTo(t).Interface().(T))
currDepth, currLength := dec.tokens.depthLength()
if err == nil && (prevDepth != currDepth || prevLength+1 != currLength) {
err = errors.New("must read exactly one JSON value")
}
if err != nil {
if err == SkipFunc {
if prevDepth == currDepth && prevLength == currLength {
return SkipFunc
}
err = errors.New("must not read any JSON tokens when skipping")
}
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
return nil
},
maySkip: true,
}
return &Unmarshalers{fncVals: []typedUnmarshaler{typFnc}, fromAny: castableToFromAny(t)}
}
// assertCastableTo asserts that "to" is a valid type to be casted to.
// These are the Go types that type-specific arshalers may operate upon.
//
// Let AllTypes be the universal set of all possible Go types.
// This function generally asserts that:
//
// len([from for from in AllTypes if castableTo(from, to)]) > 0
//
// otherwise it panics.
//
// As a special-case if marshal is false, then we forbid any non-pointer or
// non-interface type since it is almost always a bug trying to unmarshal
// into something where the end-user caller did not pass in an addressable value
// since they will not observe the mutations.
func assertCastableTo(to reflect.Type, marshal bool) {
switch to.Kind() {
case reflect.Interface:
return
case reflect.Pointer:
// Only allow unnamed pointers to be consistent with the fact that
// taking the address of a value produces an unnamed pointer type.
if to.Name() == "" {
return
}
default:
// Technically, non-pointer types are permissible for unmarshal.
// However, they are often a bug since the receiver would be immutable.
// Thus, only allow them for marshaling.
if marshal {
return
}
}
if marshal {
panic(fmt.Sprintf("input type %v must be an interface type, an unnamed pointer type, or a non-pointer type", to))
} else {
panic(fmt.Sprintf("input type %v must be an interface type or an unnamed pointer type", to))
}
}
// castableTo checks whether values of type "from" can be casted to type "to".
// Nil pointer or interface "from" values are never considered castable.
//
// This function must be kept in sync with addressableValue.castTo.
func castableTo(from, to reflect.Type) bool {
switch to.Kind() {
case reflect.Interface:
// TODO: This breaks when ordinary interfaces can have type sets
// since interfaces now exist where only the value form of a type (T)
// implements the interface, but not the pointer variant (*T).
// See https://go.dev/issue/45346.
return reflect.PointerTo(from).Implements(to)
case reflect.Pointer:
// Common case for unmarshaling.
// From must be a concrete or interface type.
return reflect.PointerTo(from) == to
default:
// Common case for marshaling.
// From must be a concrete type.
return from == to
}
}
// castTo casts va to the specified type.
// If the type is an interface, then the underlying type will always
// be a non-nil pointer to a concrete type.
//
// Requirement: castableTo(va.Type(), to) must hold.
func (va addressableValue) castTo(to reflect.Type) reflect.Value {
switch to.Kind() {
case reflect.Interface:
return va.Addr().Convert(to)
case reflect.Pointer:
return va.Addr()
default:
return va.Value
}
}
// castableToFromAny reports whether "to" can be casted to from any
// of the dynamic types used to represent arbitrary JSON.
func castableToFromAny(to reflect.Type) bool {
for _, from := range []reflect.Type{anyType, boolType, stringType, float64Type, mapStringAnyType, sliceAnyType} {
if castableTo(from, to) {
return true
}
}
return false
}
func wrapSkipFunc(err error, what string) error {
if err == SkipFunc {
return errors.New(what + " cannot be skipped")
}
return err
}

View File

@ -0,0 +1,186 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"reflect"
)
// This package supports "inlining" a Go struct field, where the contents
// of the serialized field (which must be a JSON object) are treated as if
// they are part of the parent Go struct (which represents a JSON object).
//
// Generally, inlined fields are of a Go struct type, where the fields of the
// nested struct are virtually hoisted up to the parent struct using rules
// similar to how Go embedding works (but operating within the JSON namespace).
//
// However, inlined fields may also be of a Go map type with a string key
// or a RawValue. Such inlined fields are called "fallback" fields since they
// represent any arbitrary JSON object member. Explicitly named fields take
// precedence over the inlined fallback. Only one inlined fallback is allowed.
var rawValueType = reflect.TypeOf((*RawValue)(nil)).Elem()
// marshalInlinedFallbackAll marshals all the members in an inlined fallback.
func marshalInlinedFallbackAll(mo MarshalOptions, enc *Encoder, va addressableValue, f *structField, insertUnquotedName func([]byte) bool) error {
v := addressableValue{va.Field(f.index[0])} // addressable if struct value is addressable
if len(f.index) > 1 {
v = v.fieldByIndex(f.index[1:], false)
if !v.IsValid() {
return nil // implies a nil inlined field
}
}
v = v.indirect(false)
if !v.IsValid() {
return nil
}
if v.Type() == rawValueType {
b := v.Interface().(RawValue)
if len(b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
return nil
}
dec := getBufferedDecoder(b, DecodeOptions{AllowDuplicateNames: true, AllowInvalidUTF8: true})
defer putBufferedDecoder(dec)
tok, err := dec.ReadToken()
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if tok.Kind() != '{' {
err := errors.New("inlined raw value must be a JSON object")
return &SemanticError{action: "marshal", JSONKind: tok.Kind(), GoType: rawValueType, Err: err}
}
for dec.PeekKind() != '}' {
// Parse the JSON object name.
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if insertUnquotedName != nil {
name := unescapeStringMayCopy(val, flags.isVerbatim())
if !insertUnquotedName(name) {
return &SyntacticError{str: "duplicate name " + string(val) + " in object"}
}
}
if err := enc.WriteValue(val); err != nil {
return err
}
// Parse the JSON object value.
val, err = dec.readValue(&flags)
if err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if err := enc.WriteValue(val); err != nil {
return err
}
}
if _, err := dec.ReadToken(); err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
if err := dec.checkEOF(); err != nil {
return &SemanticError{action: "marshal", GoType: rawValueType, Err: err}
}
return nil
} else {
if v.Len() == 0 {
return nil
}
m := v
mv := newAddressableValue(m.Type().Elem())
for iter := m.MapRange(); iter.Next(); {
b, err := appendString(enc.UnusedBuffer(), iter.Key().String(), !enc.options.AllowInvalidUTF8, nil)
if err != nil {
return err
}
if insertUnquotedName != nil {
isVerbatim := consumeSimpleString(b) == len(b)
name := unescapeStringMayCopy(b, isVerbatim)
if !insertUnquotedName(name) {
return &SyntacticError{str: "duplicate name " + string(b) + " in object"}
}
}
if err := enc.WriteValue(b); err != nil {
return err
}
mv.Set(iter.Value())
marshal := f.fncs.marshal
if mo.Marshalers != nil {
marshal, _ = mo.Marshalers.lookup(marshal, mv.Type())
}
if err := marshal(mo, enc, mv); err != nil {
return err
}
}
return nil
}
}
// unmarshalInlinedFallbackNext unmarshals only the next member in an inlined fallback.
func unmarshalInlinedFallbackNext(uo UnmarshalOptions, dec *Decoder, va addressableValue, f *structField, quotedName, unquotedName []byte) error {
v := addressableValue{va.Field(f.index[0])} // addressable if struct value is addressable
if len(f.index) > 1 {
v = v.fieldByIndex(f.index[1:], true)
}
v = v.indirect(true)
if v.Type() == rawValueType {
b := v.Addr().Interface().(*RawValue)
if len(*b) == 0 { // TODO: Should this be nil? What if it were all whitespace?
*b = append(*b, '{')
} else {
*b = trimSuffixWhitespace(*b)
if hasSuffixByte(*b, '}') {
// TODO: When merging into an object for the first time,
// should we verify that it is valid?
*b = trimSuffixByte(*b, '}')
*b = trimSuffixWhitespace(*b)
if !hasSuffixByte(*b, ',') && !hasSuffixByte(*b, '{') {
*b = append(*b, ',')
}
} else {
err := errors.New("inlined raw value must be a JSON object")
return &SemanticError{action: "unmarshal", GoType: rawValueType, Err: err}
}
}
*b = append(*b, quotedName...)
*b = append(*b, ':')
rawValue, err := dec.ReadValue()
if err != nil {
return err
}
*b = append(*b, rawValue...)
*b = append(*b, '}')
return nil
} else {
name := string(unquotedName) // TODO: Intern this?
m := v
if m.IsNil() {
m.Set(reflect.MakeMap(m.Type()))
}
mk := reflect.ValueOf(name)
mv := newAddressableValue(v.Type().Elem()) // TODO: Cache across calls?
if v2 := m.MapIndex(mk); v2.IsValid() {
mv.Set(v2)
}
unmarshal := f.fncs.unmarshal
if uo.Unmarshalers != nil {
unmarshal, _ = uo.Unmarshalers.lookup(unmarshal, mv.Type())
}
err := unmarshal(uo, dec, mv)
m.SetMapIndex(mk, mv.Value)
if err != nil {
return err
}
return nil
}
}

View File

@ -0,0 +1,229 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"encoding"
"errors"
"reflect"
)
// Interfaces for custom serialization.
var (
jsonMarshalerV1Type = reflect.TypeOf((*MarshalerV1)(nil)).Elem()
jsonMarshalerV2Type = reflect.TypeOf((*MarshalerV2)(nil)).Elem()
jsonUnmarshalerV1Type = reflect.TypeOf((*UnmarshalerV1)(nil)).Elem()
jsonUnmarshalerV2Type = reflect.TypeOf((*UnmarshalerV2)(nil)).Elem()
textMarshalerType = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
)
// MarshalerV1 is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerV2 unless
// the implementation is trying to avoid a hard dependency on this package.
//
// It is recommended that implementations return a buffer that is safe
// for the caller to retain and potentially mutate.
type MarshalerV1 interface {
MarshalJSON() ([]byte, error)
}
// MarshalerV2 is implemented by types that can marshal themselves.
// It is recommended that types implement MarshalerV2 instead of MarshalerV1
// since this is both more performant and flexible.
// If a type implements both MarshalerV1 and MarshalerV2,
// then MarshalerV2 takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default marshal options.
//
// The implementation must write only one JSON value to the Encoder and
// must not retain the pointer to Encoder.
type MarshalerV2 interface {
MarshalNextJSON(MarshalOptions, *Encoder) error
// TODO: Should users call the MarshalOptions.MarshalNext method or
// should/can they call this method directly? Does it matter?
}
// UnmarshalerV1 is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerV2 unless
// the implementation is trying to avoid a hard dependency on this package.
//
// The input can be assumed to be a valid encoding of a JSON value
// if called from unmarshal functionality in this package.
// UnmarshalJSON must copy the JSON data if it is retained after returning.
// It is recommended that UnmarshalJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain or mutate the input []byte.
type UnmarshalerV1 interface {
UnmarshalJSON([]byte) error
}
// UnmarshalerV2 is implemented by types that can unmarshal themselves.
// It is recommended that types implement UnmarshalerV2 instead of UnmarshalerV1
// since this is both more performant and flexible.
// If a type implements both UnmarshalerV1 and UnmarshalerV2,
// then UnmarshalerV2 takes precedence. In such a case, both implementations
// should aim to have equivalent behavior for the default unmarshal options.
//
// The implementation must read only one JSON value from the Decoder.
// It is recommended that UnmarshalNextJSON implement merge semantics when
// unmarshaling into a pre-populated value.
//
// Implementations must not retain the pointer to Decoder.
type UnmarshalerV2 interface {
UnmarshalNextJSON(UnmarshalOptions, *Decoder) error
// TODO: Should users call the UnmarshalOptions.UnmarshalNext method or
// should/can they call this method directly? Does it matter?
}
func makeMethodArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Avoid injecting method arshaler on the pointer or interface version
// to avoid ever calling the method on a nil pointer or interface receiver.
// Let it be injected on the value receiver (which is always addressable).
if t.Kind() == reflect.Pointer || t.Kind() == reflect.Interface {
return fncs
}
// Handle custom marshaler.
switch which, needAddr := implementsWhich(t, jsonMarshalerV2Type, jsonMarshalerV1Type, textMarshalerType); which {
case jsonMarshalerV2Type:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
prevDepth, prevLength := enc.tokens.depthLength()
err := va.addrWhen(needAddr).Interface().(MarshalerV2).MarshalNextJSON(mo, enc)
currDepth, currLength := enc.tokens.depthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errors.New("must write exactly one JSON value")
}
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
return nil
}
case jsonMarshalerV1Type:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
marshaler := va.addrWhen(needAddr).Interface().(MarshalerV1)
val, err := marshaler.MarshalJSON()
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping semantic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: RawValue(val).Kind(), GoType: t, Err: err}
}
return nil
}
case textMarshalerType:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
marshaler := va.addrWhen(needAddr).Interface().(encoding.TextMarshaler)
s, err := marshaler.MarshalText()
if err != nil {
err = wrapSkipFunc(err, "marshal method")
// TODO: Avoid wrapping semantic errors.
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
val := enc.UnusedBuffer()
val, err = appendString(val, string(s), true, nil)
if err != nil {
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
if err := enc.WriteValue(val); err != nil {
// TODO: Avoid wrapping syntactic or I/O errors.
return &SemanticError{action: "marshal", JSONKind: '"', GoType: t, Err: err}
}
return nil
}
}
// Handle custom unmarshaler.
switch which, needAddr := implementsWhich(t, jsonUnmarshalerV2Type, jsonUnmarshalerV1Type, textUnmarshalerType); which {
case jsonUnmarshalerV2Type:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
prevDepth, prevLength := dec.tokens.depthLength()
err := va.addrWhen(needAddr).Interface().(UnmarshalerV2).UnmarshalNextJSON(uo, dec)
currDepth, currLength := dec.tokens.depthLength()
if (prevDepth != currDepth || prevLength+1 != currLength) && err == nil {
err = errors.New("must read exactly one JSON value")
}
if err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
return nil
}
case jsonUnmarshalerV1Type:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
val, err := dec.ReadValue()
if err != nil {
return err // must be a syntactic or I/O error
}
unmarshaler := va.addrWhen(needAddr).Interface().(UnmarshalerV1)
if err := unmarshaler.UnmarshalJSON(val); err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
}
case textUnmarshalerType:
fncs.nonDefault = true
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
var flags valueFlags
val, err := dec.readValue(&flags)
if err != nil {
return err // must be a syntactic or I/O error
}
if val.Kind() != '"' {
err = errors.New("JSON value must be string type")
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
s := unescapeStringMayCopy(val, flags.isVerbatim())
unmarshaler := va.addrWhen(needAddr).Interface().(encoding.TextUnmarshaler)
if err := unmarshaler.UnmarshalText(s); err != nil {
err = wrapSkipFunc(err, "unmarshal method")
// TODO: Avoid wrapping semantic, syntactic, or I/O errors.
return &SemanticError{action: "unmarshal", JSONKind: val.Kind(), GoType: t, Err: err}
}
return nil
}
}
return fncs
}
// implementsWhich is like t.Implements(ifaceType) for a list of interfaces,
// but checks whether either t or reflect.PointerTo(t) implements the interface.
// It returns the first interface type that matches and whether a value of t
// needs to be addressed first before it implements the interface.
func implementsWhich(t reflect.Type, ifaceTypes ...reflect.Type) (which reflect.Type, needAddr bool) {
for _, ifaceType := range ifaceTypes {
switch {
case t.Implements(ifaceType):
return ifaceType, false
case reflect.PointerTo(t).Implements(ifaceType):
return ifaceType, true
}
}
return nil, false
}
// addrWhen returns va.Addr if addr is specified, otherwise it returns itself.
func (va addressableValue) addrWhen(addr bool) reflect.Value {
if addr {
return va.Addr()
}
return va.Value
}

View File

@ -0,0 +1,196 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"fmt"
"reflect"
"strings"
"time"
)
var (
timeDurationType = reflect.TypeOf((*time.Duration)(nil)).Elem()
timeTimeType = reflect.TypeOf((*time.Time)(nil)).Elem()
)
func makeTimeArshaler(fncs *arshaler, t reflect.Type) *arshaler {
// Ideally, time types would implement MarshalerV2 and UnmarshalerV2,
// but that would incur a dependency on package json from package time.
// Given how widely used time is, it is more acceptable that we incur a
// dependency on time from json.
//
// Injecting the arshaling functionality like this will not be identical
// to actually declaring methods on the time types since embedding of the
// time types will not be able to forward this functionality.
switch t {
case timeDurationType:
fncs.nonDefault = true
marshalNanos := fncs.marshal
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
if mo.format != "" && mo.formatDepth == enc.tokens.depth() {
if mo.format == "nanos" {
mo.format = ""
return marshalNanos(mo, enc, va)
} else {
return newInvalidFormatError("marshal", t, mo.format)
}
}
td := va.Interface().(time.Duration)
b := enc.UnusedBuffer()
b = append(b, '"')
b = append(b, td.String()...) // never contains special characters
b = append(b, '"')
return enc.WriteValue(b)
}
unmarshalNanos := fncs.unmarshal
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
// TODO: Should there be a flag that specifies that we can unmarshal
// from either form since there would be no ambiguity?
if uo.format != "" && uo.formatDepth == dec.tokens.depth() {
if uo.format == "nanos" {
uo.format = ""
return unmarshalNanos(uo, dec, va)
} else {
return newInvalidFormatError("unmarshal", t, uo.format)
}
}
var flags valueFlags
td := va.Addr().Interface().(*time.Duration)
val, err := dec.readValue(&flags)
if err != nil {
return err
}
switch k := val.Kind(); k {
case 'n':
*td = time.Duration(0)
return nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
td2, err := time.ParseDuration(string(val))
if err != nil {
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
}
*td = td2
return nil
default:
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t}
}
}
case timeTimeType:
fncs.nonDefault = true
fncs.marshal = func(mo MarshalOptions, enc *Encoder, va addressableValue) error {
format := time.RFC3339Nano
if mo.format != "" && mo.formatDepth == enc.tokens.depth() {
var err error
format, err = checkTimeFormat(mo.format)
if err != nil {
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
}
tt := va.Interface().(time.Time)
if y := tt.Year(); y < 0 || y >= 10000 {
// RFC 3339 is clear that years are 4 digits exactly.
// See https://go.dev/issue/4556#c15 for more discussion.
err := fmt.Errorf("year %d outside of range [0,9999]", y)
return &SemanticError{action: "marshal", GoType: t, Err: err}
}
b := enc.UnusedBuffer()
b = append(b, '"')
b = tt.AppendFormat(b, format)
b = append(b, '"')
// The format may contain special characters that need escaping.
// Verify that the result is a valid JSON string (common case),
// otherwise escape the string correctly (slower case).
if consumeSimpleString(b) != len(b) {
b, _ = appendString(nil, string(b[len(`"`):len(b)-len(`"`)]), true, nil)
}
return enc.WriteValue(b)
}
fncs.unmarshal = func(uo UnmarshalOptions, dec *Decoder, va addressableValue) error {
format := time.RFC3339Nano
if uo.format != "" && uo.formatDepth == dec.tokens.depth() {
var err error
format, err = checkTimeFormat(uo.format)
if err != nil {
return &SemanticError{action: "unmarshal", GoType: t, Err: err}
}
}
var flags valueFlags
tt := va.Addr().Interface().(*time.Time)
val, err := dec.readValue(&flags)
if err != nil {
return err
}
k := val.Kind()
switch k {
case 'n':
*tt = time.Time{}
return nil
case '"':
val = unescapeStringMayCopy(val, flags.isVerbatim())
tt2, err := time.Parse(format, string(val))
if err != nil {
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t, Err: err}
}
*tt = tt2
return nil
default:
return &SemanticError{action: "unmarshal", JSONKind: k, GoType: t}
}
}
}
return fncs
}
func checkTimeFormat(format string) (string, error) {
// We assume that an exported constant in the time package will
// always start with an uppercase ASCII letter.
if len(format) > 0 && 'A' <= format[0] && format[0] <= 'Z' {
switch format {
case "ANSIC":
return time.ANSIC, nil
case "UnixDate":
return time.UnixDate, nil
case "RubyDate":
return time.RubyDate, nil
case "RFC822":
return time.RFC822, nil
case "RFC822Z":
return time.RFC822Z, nil
case "RFC850":
return time.RFC850, nil
case "RFC1123":
return time.RFC1123, nil
case "RFC1123Z":
return time.RFC1123Z, nil
case "RFC3339":
return time.RFC3339, nil
case "RFC3339Nano":
return time.RFC3339Nano, nil
case "Kitchen":
return time.Kitchen, nil
case "Stamp":
return time.Stamp, nil
case "StampMilli":
return time.StampMilli, nil
case "StampMicro":
return time.StampMicro, nil
case "StampNano":
return time.StampNano, nil
default:
// Reject any format that is an exported Go identifier in case
// new format constants are added to the time package.
if strings.TrimFunc(format, isLetterOrDigit) == "" {
return "", fmt.Errorf("undefined format layout: %v", format)
}
}
}
return format, nil
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,185 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package json implements serialization of JSON
// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
//
// Terminology
//
// This package uses the terms "encode" and "decode" for syntactic functionality
// that is concerned with processing JSON based on its grammar, and
// uses the terms "marshal" and "unmarshal" for semantic functionality
// that determines the meaning of JSON values as Go values and vice-versa.
// It aims to provide a clear distinction between functionality that
// is purely concerned with encoding versus that of marshaling.
// For example, one can directly encode a stream of JSON tokens without
// needing to marshal a concrete Go value representing them.
// Similarly, one can decode a stream of JSON tokens without
// needing to unmarshal them into a concrete Go value.
//
// This package uses JSON terminology when discussing JSON, which may differ
// from related concepts in Go or elsewhere in computing literature.
//
// - A JSON "object" refers to an unordered collection of name/value members.
// - A JSON "array" refers to an ordered sequence of elements.
// - A JSON "value" refers to either a literal (i.e., null, false, or true),
// string, number, object, or array.
//
// See RFC 8259 for more information.
//
//
// Specifications
//
// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259,
// and RFC 8785. Each RFC is generally a stricter subset of another RFC.
// In increasing order of strictness:
//
// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8
// and also do not require (but recommend) that object names be unique.
// - RFC 8259 requires the use of UTF-8,
// but does not require (but recommends) that object names be unique.
// - RFC 7493 requires the use of UTF-8
// and also requires that object names be unique.
// - RFC 8785 defines a canonical representation. It requires the use of UTF-8
// and also requires that object names be unique and in a specific ordering.
// It specifies exactly how strings and numbers must be formatted.
//
// The primary difference between RFC 4627 and RFC 7159 is that the former
// restricted top-level values to only JSON objects and arrays, while
// RFC 7159 and subsequent RFCs permit top-level values to additionally be
// JSON nulls, booleans, strings, or numbers.
//
// By default, this package operates on RFC 7493, but can be configured
// to operate according to the other RFC specifications.
// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it.
// In particular, it makes specific choices about behavior that RFC 8259
// leaves as undefined in order to ensure greater interoperability.
//
//
// JSON Representation of Go structs
//
// A Go struct is naturally represented as a JSON object,
// where each Go struct field corresponds with a JSON object member.
// When marshaling, all Go struct fields are recursively encoded in depth-first
// order as JSON object members except those that are ignored or omitted.
// When unmarshaling, JSON object members are recursively decoded
// into the corresponding Go struct fields.
// Object members that do not match any struct fields,
// also known as “unknown members”, are ignored by default or rejected
// if UnmarshalOptions.RejectUnknownMembers is specified.
//
// The representation of each struct field can be customized in the
// "json" struct field tag, where the tag is a comma separated list of options.
// As a special case, if the entire tag is `json:"-"`,
// then the field is ignored with regard to its JSON representation.
//
// The first option is the JSON object name override for the Go struct field.
// If the name is not specified, then the Go struct field name
// is used as the JSON object name. JSON names containing commas or quotes,
// or names identical to "" or "-", can be specified using
// a single-quoted string literal, where the syntax is identical to
// the Go grammar for a double-quoted string literal,
// but instead uses single quotes as the delimiters.
// By default, unmarshaling uses case-sensitive matching to identify
// the Go struct field associated with a JSON object name.
//
// After the name, the following tag options are supported:
//
// - omitzero: When marshaling, the "omitzero" option specifies that
// the struct field should be omitted if the field value is zero
// as determined by the "IsZero() bool" method if present,
// otherwise based on whether the field is the zero Go value.
// This option has no effect when unmarshaling.
//
// - omitempty: When marshaling, the "omitempty" option specifies that
// the struct field should be omitted if the field value would have been
// encoded as a JSON null, empty string, empty object, or empty array.
// This option has no effect when unmarshaling.
//
// - string: The "string" option specifies that
// MarshalOptions.StringifyNumbers and UnmarshalOptions.StringifyNumbers
// be set when marshaling or unmarshaling a struct field value.
// This causes numeric types to be encoded as a JSON number
// within a JSON string, and to be decoded from either a JSON number or
// a JSON string containing a JSON number.
// This extra level of encoding is often necessary since
// many JSON parsers cannot precisely represent 64-bit integers.
//
// - nocase: When unmarshaling, the "nocase" option specifies that
// if the JSON object name does not exactly match the JSON name
// for any of the struct fields, then it attempts to match the struct field
// using a case-insensitive match that also ignores dashes and underscores.
// If multiple fields match, the first declared field in breadth-first order
// takes precedence. This option has no effect when marshaling.
//
// - inline: The "inline" option specifies that
// the JSON representable content of this field type is to be promoted
// as if they were specified in the parent struct.
// It is the JSON equivalent of Go struct embedding.
// A Go embedded field is implicitly inlined unless an explicit JSON name
// is specified. The inlined field must be a Go struct
// (that does not implement any JSON methods), RawValue, map[string]T,
// or an unnamed pointer to such types. When marshaling,
// inlined fields from a pointer type are omitted if it is nil.
// Inlined fields of type RawValue and map[string]T are called
// “inlined fallbacks” as they can represent all possible
// JSON object members not directly handled by the parent struct.
// Only one inlined fallback field may be specified in a struct,
// while many non-fallback fields may be specified. This option
// must not be specified with any other option (including the JSON name).
//
// - unknown: The "unknown" option is a specialized variant
// of the inlined fallback to indicate that this Go struct field
// contains any number of unknown JSON object members. The field type
// must be a RawValue, map[string]T, or an unnamed pointer to such types.
// If MarshalOptions.DiscardUnknownMembers is specified when marshaling,
// the contents of this field are ignored.
// If UnmarshalOptions.RejectUnknownMembers is specified when unmarshaling,
// any unknown object members are rejected regardless of whether
// an inlined fallback with the "unknown" option exists. This option
// must not be specified with any other option (including the JSON name).
//
// - format: The "format" option specifies a format flag
// used to specialize the formatting of the field value.
// The option is a key-value pair specified as "format:value" where
// the value must be either a literal consisting of letters and numbers
// (e.g., "format:RFC3339") or a single-quoted string literal
// (e.g., "format:'2006-01-02'"). The interpretation of the format flag
// is determined by the struct field type.
//
// The "omitzero" and "omitempty" options are mostly semantically identical.
// The former is defined in terms of the Go type system,
// while the latter in terms of the JSON type system.
// Consequently they behave differently in some circumstances.
// For example, only a nil slice or map is omitted under "omitzero", while
// an empty slice or map is omitted under "omitempty" regardless of nilness.
// The "omitzero" option is useful for types with a well-defined zero value
// (e.g., netip.Addr) or have an IsZero method (e.g., time.Time).
//
// Every Go struct corresponds to a list of JSON representable fields
// which is constructed by performing a breadth-first search over
// all struct fields (excluding unexported or ignored fields),
// where the search recursively descends into inlined structs.
// The set of non-inlined fields in a struct must have unique JSON names.
// If multiple fields all have the same JSON name, then the one
// at shallowest depth takes precedence and the other fields at deeper depths
// are excluded from the list of JSON representable fields.
// If multiple fields at the shallowest depth have the same JSON name,
// then all of those fields are excluded from the list. This is analogous to
// Go visibility rules for struct field selection with embedded struct types.
//
// Marshaling or unmarshaling a non-empty struct
// without any JSON representable fields results in a SemanticError.
// Unexported fields must not have any `json` tags except for `json:"-"`.
package json
// requireKeyedLiterals can be embedded in a struct to require keyed literals.
type requireKeyedLiterals struct{}
// nonComparable can be embedded in a struct to prevent comparability.
type nonComparable [0]func()

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,183 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"reflect"
"strconv"
"strings"
"unicode/utf8"
)
const errorPrefix = "json: "
// Error matches errors returned by this package according to errors.Is.
const Error = jsonError("json error")
type jsonError string
func (e jsonError) Error() string {
return string(e)
}
func (e jsonError) Is(target error) bool {
return e == target || target == Error
}
type ioError struct {
action string // either "read" or "write"
err error
}
func (e *ioError) Error() string {
return errorPrefix + e.action + " error: " + e.err.Error()
}
func (e *ioError) Unwrap() error {
return e.err
}
func (e *ioError) Is(target error) bool {
return e == target || target == Error || errors.Is(e.err, target)
}
// SemanticError describes an error determining the meaning
// of JSON data as Go data or vice-versa.
//
// The contents of this error as produced by this package may change over time.
type SemanticError struct {
requireKeyedLiterals
nonComparable
action string // either "marshal" or "unmarshal"
// ByteOffset indicates that an error occurred after this byte offset.
ByteOffset int64
// JSONPointer indicates that an error occurred within this JSON value
// as indicated using the JSON Pointer notation (see RFC 6901).
JSONPointer string
// JSONKind is the JSON kind that could not be handled.
JSONKind Kind // may be zero if unknown
// GoType is the Go type that could not be handled.
GoType reflect.Type // may be nil if unknown
// Err is the underlying error.
Err error // may be nil
}
func (e *SemanticError) Error() string {
var sb strings.Builder
sb.WriteString(errorPrefix)
// Hyrum-proof the error message by deliberately switching between
// two equivalent renderings of the same error message.
// The randomization is tied to the Hyrum-proofing already applied
// on map iteration in Go.
for phrase := range map[string]struct{}{"cannot": {}, "unable to": {}} {
sb.WriteString(phrase)
break // use whichever phrase we get in the first iteration
}
// Format action.
var preposition string
switch e.action {
case "marshal":
sb.WriteString(" marshal")
preposition = " from"
case "unmarshal":
sb.WriteString(" unmarshal")
preposition = " into"
default:
sb.WriteString(" handle")
preposition = " with"
}
// Format JSON kind.
var omitPreposition bool
switch e.JSONKind {
case 'n':
sb.WriteString(" JSON null")
case 'f', 't':
sb.WriteString(" JSON boolean")
case '"':
sb.WriteString(" JSON string")
case '0':
sb.WriteString(" JSON number")
case '{', '}':
sb.WriteString(" JSON object")
case '[', ']':
sb.WriteString(" JSON array")
default:
omitPreposition = true
}
// Format Go type.
if e.GoType != nil {
if !omitPreposition {
sb.WriteString(preposition)
}
sb.WriteString(" Go value of type ")
sb.WriteString(e.GoType.String())
}
// Format where.
switch {
case e.JSONPointer != "":
sb.WriteString(" within JSON value at ")
sb.WriteString(strconv.Quote(e.JSONPointer))
case e.ByteOffset > 0:
sb.WriteString(" after byte offset ")
sb.WriteString(strconv.FormatInt(e.ByteOffset, 10))
}
// Format underlying error.
if e.Err != nil {
sb.WriteString(": ")
sb.WriteString(e.Err.Error())
}
return sb.String()
}
func (e *SemanticError) Is(target error) bool {
return e == target || target == Error || errors.Is(e.Err, target)
}
func (e *SemanticError) Unwrap() error {
return e.Err
}
// SyntacticError is a description of a syntactic error that occurred when
// encoding or decoding JSON according to the grammar.
//
// The contents of this error as produced by this package may change over time.
type SyntacticError struct {
requireKeyedLiterals
nonComparable
// ByteOffset indicates that an error occurred after this byte offset.
ByteOffset int64
str string
}
func (e *SyntacticError) Error() string {
return errorPrefix + e.str
}
func (e *SyntacticError) Is(target error) bool {
return e == target || target == Error
}
func (e *SyntacticError) withOffset(pos int64) error {
return &SyntacticError{ByteOffset: pos, str: e.str}
}
func newInvalidCharacterError(prefix []byte, where string) *SyntacticError {
what := quoteRune(prefix)
return &SyntacticError{str: "invalid character " + what + " " + where}
}
func quoteRune(b []byte) string {
r, n := utf8.DecodeRune(b)
if r == utf8.RuneError && n == 1 {
return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'`
}
return strconv.QuoteRune(r)
}

View File

@ -0,0 +1,509 @@
// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"errors"
"fmt"
"io"
"reflect"
"sort"
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
var errIgnoredField = errors.New("ignored field")
type isZeroer interface {
IsZero() bool
}
var isZeroerType = reflect.TypeOf((*isZeroer)(nil)).Elem()
type structFields struct {
flattened []structField // listed in depth-first ordering
byActualName map[string]*structField
byFoldedName map[string][]*structField
inlinedFallback *structField
}
type structField struct {
id int // unique numeric ID in breadth-first ordering
index []int // index into a struct according to reflect.Type.FieldByIndex
typ reflect.Type
fncs *arshaler
isZero func(addressableValue) bool
isEmpty func(addressableValue) bool
fieldOptions
}
func makeStructFields(root reflect.Type) (structFields, *SemanticError) {
var fs structFields
fs.byActualName = make(map[string]*structField, root.NumField())
fs.byFoldedName = make(map[string][]*structField, root.NumField())
// ambiguous is a sentinel value to indicate that at least two fields
// at the same depth have the same name, and thus cancel each other out.
// This follows the same rules as selecting a field on embedded structs
// where the shallowest field takes precedence. If more than one field
// exists at the shallowest depth, then the selection is illegal.
// See https://go.dev/ref/spec#Selectors.
ambiguous := new(structField)
// Setup a queue for a breath-first search.
var queueIndex int
type queueEntry struct {
typ reflect.Type
index []int
visitChildren bool // whether to recursively visit inlined field in this struct
}
queue := []queueEntry{{root, nil, true}}
seen := map[reflect.Type]bool{root: true}
// Perform a breadth-first search over all reachable fields.
// This ensures that len(f.index) will be monotonically increasing.
for queueIndex < len(queue) {
qe := queue[queueIndex]
queueIndex++
t := qe.typ
inlinedFallbackIndex := -1 // index of last inlined fallback field in current struct
namesIndex := make(map[string]int) // index of each field with a given JSON object name in current struct
var hasAnyJSONTag bool // whether any Go struct field has a `json` tag
var hasAnyJSONField bool // whether any JSON serializable fields exist in current struct
for i := 0; i < t.NumField(); i++ {
sf := t.Field(i)
_, hasTag := sf.Tag.Lookup("json")
hasAnyJSONTag = hasAnyJSONTag || hasTag
options, err := parseFieldOptions(sf)
if err != nil {
if err == errIgnoredField {
continue
}
return structFields{}, &SemanticError{GoType: t, Err: err}
}
hasAnyJSONField = true
f := structField{
// Allocate a new slice (len=N+1) to hold both
// the parent index (len=N) and the current index (len=1).
// Do this to avoid clobbering the memory of the parent index.
index: append(append(make([]int, 0, len(qe.index)+1), qe.index...), i),
typ: sf.Type,
fieldOptions: options,
}
if sf.Anonymous && !f.hasName {
f.inline = true // implied by use of Go embedding without an explicit name
}
if f.inline || f.unknown {
// Handle an inlined field that serializes to/from
// zero or more JSON object members.
if f.inline && f.unknown {
err := fmt.Errorf("Go struct field %s cannot have both `inline` and `unknown` specified", sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
switch f.fieldOptions {
case fieldOptions{name: f.name, quotedName: f.quotedName, inline: true}:
case fieldOptions{name: f.name, quotedName: f.quotedName, unknown: true}:
default:
err := fmt.Errorf("Go struct field %s cannot have any options other than `inline` or `unknown` specified", sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Unwrap one level of pointer indirection similar to how Go
// only allows embedding either T or *T, but not **T.
tf := f.typ
if tf.Kind() == reflect.Pointer && tf.Name() == "" {
tf = tf.Elem()
}
// Reject any types with custom serialization otherwise
// it becomes impossible to know what sub-fields to inline.
if which, _ := implementsWhich(tf,
jsonMarshalerV2Type, jsonMarshalerV1Type, textMarshalerType,
jsonUnmarshalerV2Type, jsonUnmarshalerV1Type, textUnmarshalerType,
); which != nil && tf != rawValueType {
err := fmt.Errorf("inlined Go struct field %s of type %s must not implement JSON marshal or unmarshal methods", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Handle an inlined field that serializes to/from
// a finite number of JSON object members backed by a Go struct.
if tf.Kind() == reflect.Struct {
if f.unknown {
err := fmt.Errorf("inlined Go struct field %s of type %s with `unknown` tag must be a Go map of string key or a json.RawValue", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
if qe.visitChildren {
queue = append(queue, queueEntry{tf, f.index, !seen[tf]})
}
seen[tf] = true
continue
}
// Handle an inlined field that serializes to/from any number of
// JSON object members back by a Go map or RawValue.
switch {
case tf == rawValueType:
f.fncs = nil // specially handled in arshal_inlined.go
case tf.Kind() == reflect.Map && tf.Key() == stringType:
f.fncs = lookupArshaler(tf.Elem())
default:
err := fmt.Errorf("inlined Go struct field %s of type %s must be a Go struct, Go map of string key, or json.RawValue", sf.Name, tf)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Reject multiple inlined fallback fields within the same struct.
if inlinedFallbackIndex >= 0 {
err := fmt.Errorf("inlined Go struct fields %s and %s cannot both be a Go map or json.RawValue", t.Field(inlinedFallbackIndex).Name, sf.Name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
inlinedFallbackIndex = i
// Multiple inlined fallback fields across different structs
// follow the same precedence rules as Go struct embedding.
if fs.inlinedFallback == nil {
fs.inlinedFallback = &f // store first occurrence at lowest depth
} else if len(fs.inlinedFallback.index) == len(f.index) {
fs.inlinedFallback = ambiguous // at least two occurrences at same depth
}
} else {
// Handle normal Go struct field that serializes to/from
// a single JSON object member.
// Provide a function that uses a type's IsZero method.
switch {
case sf.Type.Kind() == reflect.Interface && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on a nil interface or
// non-nil interface with nil pointer.
return va.IsNil() || (va.Elem().Kind() == reflect.Pointer && va.Elem().IsNil()) || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Kind() == reflect.Pointer && sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool {
// Avoid panics calling IsZero on nil pointer.
return va.IsNil() || va.Interface().(isZeroer).IsZero()
}
case sf.Type.Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Interface().(isZeroer).IsZero() }
case reflect.PointerTo(sf.Type).Implements(isZeroerType):
f.isZero = func(va addressableValue) bool { return va.Addr().Interface().(isZeroer).IsZero() }
}
// Provide a function that can determine whether the value would
// serialize as an empty JSON value.
switch sf.Type.Kind() {
case reflect.String, reflect.Map, reflect.Array, reflect.Slice:
f.isEmpty = func(va addressableValue) bool { return va.Len() == 0 }
case reflect.Pointer, reflect.Interface:
f.isEmpty = func(va addressableValue) bool { return va.IsNil() }
}
f.id = len(fs.flattened)
f.fncs = lookupArshaler(sf.Type)
fs.flattened = append(fs.flattened, f)
// Reject user-specified names with invalid UTF-8.
if !utf8.ValidString(f.name) {
err := fmt.Errorf("Go struct field %s has JSON object name %q with invalid UTF-8", sf.Name, f.name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
// Reject multiple fields with same name within the same struct.
if j, ok := namesIndex[f.name]; ok {
err := fmt.Errorf("Go struct fields %s and %s conflict over JSON object name %q", t.Field(j).Name, sf.Name, f.name)
return structFields{}, &SemanticError{GoType: t, Err: err}
}
namesIndex[f.name] = i
// Multiple fields of the same name across different structs
// follow the same precedence rules as Go struct embedding.
if f2 := fs.byActualName[f.name]; f2 == nil {
fs.byActualName[f.name] = &fs.flattened[len(fs.flattened)-1] // store first occurrence at lowest depth
} else if len(f2.index) == len(f.index) {
fs.byActualName[f.name] = ambiguous // at least two occurrences at same depth
}
}
}
// NOTE: New users to the json package are occasionally surprised that
// unexported fields are ignored. This occurs by necessity due to our
// inability to directly introspect such fields with Go reflection
// without the use of unsafe.
//
// To reduce friction here, refuse to serialize any Go struct that
// has no JSON serializable fields, has at least one Go struct field,
// and does not have any `json` tags present. For example,
// errors returned by errors.New would fail to serialize.
isEmptyStruct := t.NumField() == 0
if !isEmptyStruct && !hasAnyJSONTag && !hasAnyJSONField {
err := errors.New("Go struct has no exported fields")
return structFields{}, &SemanticError{GoType: t, Err: err}
}
}
// Remove all fields that are duplicates.
// This may move elements forward to fill the holes from removed fields.
var n int
for _, f := range fs.flattened {
switch f2 := fs.byActualName[f.name]; {
case f2 == ambiguous:
delete(fs.byActualName, f.name)
case f2 == nil:
continue // may be nil due to previous delete
// TODO(https://go.dev/issue/45955): Use slices.Equal.
case reflect.DeepEqual(f.index, f2.index):
f.id = n
fs.flattened[n] = f
fs.byActualName[f.name] = &fs.flattened[n] // fix pointer to new location
n++
}
}
fs.flattened = fs.flattened[:n]
if fs.inlinedFallback == ambiguous {
fs.inlinedFallback = nil
}
if len(fs.flattened) != len(fs.byActualName) {
panic(fmt.Sprintf("BUG: flattened list of fields mismatches fields mapped by name: %d != %d", len(fs.flattened), len(fs.byActualName)))
}
// Sort the fields according to a depth-first ordering.
// This operation will cause pointers in byActualName to become incorrect,
// which we will correct in another loop shortly thereafter.
sort.Slice(fs.flattened, func(i, j int) bool {
si := fs.flattened[i].index
sj := fs.flattened[j].index
for len(si) > 0 && len(sj) > 0 {
switch {
case si[0] < sj[0]:
return true
case si[0] > sj[0]:
return false
default:
si = si[1:]
sj = sj[1:]
}
}
return len(si) < len(sj)
})
// Recompute the mapping of fields in the byActualName map.
// Pre-fold all names so that we can lookup folded names quickly.
for i, f := range fs.flattened {
foldedName := string(foldName([]byte(f.name)))
fs.byActualName[f.name] = &fs.flattened[i]
fs.byFoldedName[foldedName] = append(fs.byFoldedName[foldedName], &fs.flattened[i])
}
for foldedName, fields := range fs.byFoldedName {
if len(fields) > 1 {
// The precedence order for conflicting nocase names
// is by breadth-first order, rather than depth-first order.
sort.Slice(fields, func(i, j int) bool {
return fields[i].id < fields[j].id
})
fs.byFoldedName[foldedName] = fields
}
}
return fs, nil
}
type fieldOptions struct {
name string
quotedName string // quoted name per RFC 8785, section 3.2.2.2.
hasName bool
nocase bool
inline bool
unknown bool
omitzero bool
omitempty bool
string bool
format string
}
// parseFieldOptions parses the `json` tag in a Go struct field as
// a structured set of options configuring parameters such as
// the JSON member name and other features.
// As a special case, it returns errIgnoredField if the field is ignored.
func parseFieldOptions(sf reflect.StructField) (out fieldOptions, err error) {
tag, hasTag := sf.Tag.Lookup("json")
// Check whether this field is explicitly ignored.
if tag == "-" {
return fieldOptions{}, errIgnoredField
}
// Check whether this field is unexported.
if !sf.IsExported() {
// In contrast to v1, v2 no longer forwards exported fields from
// embedded fields of unexported types since Go reflection does not
// allow the same set of operations that are available in normal cases
// of purely exported fields.
// See https://go.dev/issue/21357 and https://go.dev/issue/24153.
if sf.Anonymous {
return fieldOptions{}, fmt.Errorf("embedded Go struct field %s of an unexported type must be explicitly ignored with a `json:\"-\"` tag", sf.Type.Name())
}
// Tag options specified on an unexported field suggests user error.
if hasTag {
return fieldOptions{}, fmt.Errorf("unexported Go struct field %s cannot have non-ignored `json:%q` tag", sf.Name, tag)
}
return fieldOptions{}, errIgnoredField
}
// Determine the JSON member name for this Go field. A user-specified name
// may be provided as either an identifier or a single-quoted string.
// The single-quoted string allows arbitrary characters in the name.
// See https://go.dev/issue/2718 and https://go.dev/issue/3546.
out.name = sf.Name // always starts with an uppercase character
if len(tag) > 0 && !strings.HasPrefix(tag, ",") {
// For better compatibility with v1, accept almost any unescaped name.
n := len(tag) - len(strings.TrimLeftFunc(tag, func(r rune) bool {
return !strings.ContainsRune(",\\'\"`", r) // reserve comma, backslash, and quotes
}))
opt := tag[:n]
if n == 0 {
// Allow a single quoted string for arbitrary names.
opt, n, err = consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err)
}
}
out.hasName = true
out.name = opt
tag = tag[n:]
}
b, _ := appendString(nil, out.name, false, nil)
out.quotedName = string(b)
// Handle any additional tag options (if any).
var wasFormat bool
seenOpts := make(map[string]bool)
for len(tag) > 0 {
// Consume comma delimiter.
if tag[0] != ',' {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid character %q before next option (expecting ',')", sf.Name, tag[0])
}
tag = tag[len(","):]
if len(tag) == 0 {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: invalid trailing ',' character", sf.Name)
}
// Consume and process the tag option.
opt, n, err := consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed `json` tag: %v", sf.Name, err)
}
rawOpt := tag[:n]
tag = tag[n:]
switch {
case wasFormat:
return fieldOptions{}, fmt.Errorf("Go struct field %s has `format` tag option that was not specified last", sf.Name)
case strings.HasPrefix(rawOpt, "'") && strings.TrimFunc(opt, isLetterOrDigit) == "":
return fieldOptions{}, fmt.Errorf("Go struct field %s has unnecessarily quoted appearance of `%s` tag option; specify `%s` instead", sf.Name, rawOpt, opt)
}
switch opt {
case "nocase":
out.nocase = true
case "inline":
out.inline = true
case "unknown":
out.unknown = true
case "omitzero":
out.omitzero = true
case "omitempty":
out.omitempty = true
case "string":
out.string = true
case "format":
if !strings.HasPrefix(tag, ":") {
return fieldOptions{}, fmt.Errorf("Go struct field %s is missing value for `format` tag option", sf.Name)
}
tag = tag[len(":"):]
opt, n, err := consumeTagOption(tag)
if err != nil {
return fieldOptions{}, fmt.Errorf("Go struct field %s has malformed value for `format` tag option: %v", sf.Name, err)
}
tag = tag[n:]
out.format = opt
wasFormat = true
default:
// Reject keys that resemble one of the supported options.
// This catches invalid mutants such as "omitEmpty" or "omit_empty".
normOpt := strings.ReplaceAll(strings.ToLower(opt), "_", "")
switch normOpt {
case "nocase", "inline", "unknown", "omitzero", "omitempty", "string", "format":
return fieldOptions{}, fmt.Errorf("Go struct field %s has invalid appearance of `%s` tag option; specify `%s` instead", sf.Name, opt, normOpt)
}
// NOTE: Everything else is ignored. This does not mean it is
// forward compatible to insert arbitrary tag options since
// a future version of this package may understand that tag.
}
// Reject duplicates.
if seenOpts[opt] {
return fieldOptions{}, fmt.Errorf("Go struct field %s has duplicate appearance of `%s` tag option", sf.Name, rawOpt)
}
seenOpts[opt] = true
}
return out, nil
}
func consumeTagOption(in string) (string, int, error) {
switch r, _ := utf8.DecodeRuneInString(in); {
// Option as a Go identifier.
case r == '_' || unicode.IsLetter(r):
n := len(in) - len(strings.TrimLeftFunc(in, isLetterOrDigit))
return in[:n], n, nil
// Option as a single-quoted string.
case r == '\'':
// The grammar is nearly identical to a double-quoted Go string literal,
// but uses single quotes as the terminators. The reason for a custom
// grammar is because both backtick and double quotes cannot be used
// verbatim in a struct tag.
//
// Convert a single-quoted string to a double-quote string and rely on
// strconv.Unquote to handle the rest.
var inEscape bool
b := []byte{'"'}
n := len(`'`)
for len(in) > n {
r, rn := utf8.DecodeRuneInString(in[n:])
switch {
case inEscape:
if r == '\'' {
b = b[:len(b)-1] // remove escape character: `\'` => `'`
}
inEscape = false
case r == '\\':
inEscape = true
case r == '"':
b = append(b, '\\') // insert escape character: `"` => `\"`
case r == '\'':
b = append(b, '"')
n += len(`'`)
out, err := strconv.Unquote(string(b))
if err != nil {
return "", 0, fmt.Errorf("invalid single-quoted string: %s", in[:n])
}
return out, n, nil
}
b = append(b, in[n:][:rn]...)
n += rn
}
if n > 10 {
n = 10 // limit the amount of context printed in the error
}
return "", 0, fmt.Errorf("single-quoted string not terminated: %s...", in[:n])
case len(in) == 0:
return "", 0, io.ErrUnexpectedEOF
default:
return "", 0, fmt.Errorf("invalid character %q at start of option (expecting Unicode letter or single quote)", r)
}
}
func isLetterOrDigit(r rune) bool {
return r == '_' || unicode.IsLetter(r) || unicode.IsNumber(r)
}

View File

@ -0,0 +1,56 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"unicode"
"unicode/utf8"
)
// foldName returns a folded string such that foldName(x) == foldName(y)
// is similar to strings.EqualFold(x, y), but ignores underscore and dashes.
// This allows foldName to match common naming conventions.
func foldName(in []byte) []byte {
// This is inlinable to take advantage of "function outlining".
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
var arr [32]byte // large enough for most JSON names
return appendFoldedName(arr[:0], in)
}
func appendFoldedName(out, in []byte) []byte {
for i := 0; i < len(in); {
// Handle single-byte ASCII.
if c := in[i]; c < utf8.RuneSelf {
if c != '_' && c != '-' {
if 'a' <= c && c <= 'z' {
c -= 'a' - 'A'
}
out = append(out, c)
}
i++
continue
}
// Handle multi-byte Unicode.
r, n := utf8.DecodeRune(in[i:])
out = utf8.AppendRune(out, foldRune(r))
i += n
}
return out
}
// foldRune is a variation on unicode.SimpleFold that returns the same rune
// for all runes in the same fold set.
//
// Invariant:
//
// foldRune(x) == foldRune(y) ⇔ strings.EqualFold(string(x), string(y))
func foldRune(r rune) rune {
for {
r2 := unicode.SimpleFold(r)
if r2 <= r {
return r2 // smallest character in the fold set
}
r = r2
}
}

View File

@ -0,0 +1,86 @@
// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"encoding/binary"
"math/bits"
)
// stringCache is a cache for strings converted from a []byte.
type stringCache [256]string // 256*unsafe.Sizeof(string("")) => 4KiB
// make returns the string form of b.
// It returns a pre-allocated string from c if present, otherwise
// it allocates a new string, inserts it into the cache, and returns it.
func (c *stringCache) make(b []byte) string {
const (
minCachedLen = 2 // single byte strings are already interned by the runtime
maxCachedLen = 256 // large enough for UUIDs, IPv6 addresses, SHA-256 checksums, etc.
)
if c == nil || len(b) < minCachedLen || len(b) > maxCachedLen {
return string(b)
}
// Compute a hash from the fixed-width prefix and suffix of the string.
// This ensures hashing a string is a constant time operation.
var h uint32
switch {
case len(b) >= 8:
lo := binary.LittleEndian.Uint64(b[:8])
hi := binary.LittleEndian.Uint64(b[len(b)-8:])
h = hash64(uint32(lo), uint32(lo>>32)) ^ hash64(uint32(hi), uint32(hi>>32))
case len(b) >= 4:
lo := binary.LittleEndian.Uint32(b[:4])
hi := binary.LittleEndian.Uint32(b[len(b)-4:])
h = hash64(lo, hi)
case len(b) >= 2:
lo := binary.LittleEndian.Uint16(b[:2])
hi := binary.LittleEndian.Uint16(b[len(b)-2:])
h = hash64(uint32(lo), uint32(hi))
}
// Check the cache for the string.
i := h % uint32(len(*c))
if s := (*c)[i]; s == string(b) {
return s
}
s := string(b)
(*c)[i] = s
return s
}
// hash64 returns the hash of two uint32s as a single uint32.
func hash64(lo, hi uint32) uint32 {
// If avalanche=true, this is identical to XXH32 hash on a 8B string:
// var b [8]byte
// binary.LittleEndian.PutUint32(b[:4], lo)
// binary.LittleEndian.PutUint32(b[4:], hi)
// return xxhash.Sum32(b[:])
const (
prime1 = 0x9e3779b1
prime2 = 0x85ebca77
prime3 = 0xc2b2ae3d
prime4 = 0x27d4eb2f
prime5 = 0x165667b1
)
h := prime5 + uint32(8)
h += lo * prime3
h = bits.RotateLeft32(h, 17) * prime4
h += hi * prime3
h = bits.RotateLeft32(h, 17) * prime4
// Skip final mix (avalanche) step of XXH32 for performance reasons.
// Empirical testing shows that the improvements in unbiased distribution
// does not outweigh the extra cost in computational complexity.
const avalanche = false
if avalanche {
h ^= h >> 15
h *= prime2
h ^= h >> 13
h *= prime3
h ^= h >> 16
}
return h
}

View File

@ -0,0 +1,150 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"bytes"
"io"
"math/bits"
"sync"
)
// TODO(https://go.dev/issue/47657): Use sync.PoolOf.
var (
// This owns the internal buffer since there is no io.Writer to output to.
// Since the buffer can get arbitrarily large in normal usage,
// there is statistical tracking logic to determine whether to recycle
// the internal buffer or not based on a history of utilization.
bufferedEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON before flushing it to the underlying io.Writer.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
)
// bufferStatistics is statistics to track buffer utilization.
// It is used to determine whether to recycle a buffer or not
// to avoid https://go.dev/issue/23199.
type bufferStatistics struct {
strikes int // number of times the buffer was under-utilized
prevLen int // length of previous buffer
}
func getBufferedEncoder(o EncodeOptions) *Encoder {
e := bufferedEncoderPool.Get().(*Encoder)
if e.buf == nil {
// Round up to nearest 2ⁿ to make best use of malloc size classes.
// See runtime/sizeclasses.go on Go1.15.
// Logical OR with 63 to ensure 64 as the minimum buffer size.
n := 1 << bits.Len(uint(e.bufStats.prevLen|63))
e.buf = make([]byte, 0, n)
}
e.reset(e.buf[:0], nil, o)
return e
}
func putBufferedEncoder(e *Encoder) {
// Recycle large buffers only if sufficiently utilized.
// If a buffer is under-utilized enough times sequentially,
// then it is discarded, ensuring that a single large buffer
// won't be kept alive by a continuous stream of small usages.
//
// The worst case utilization is computed as:
// MIN_UTILIZATION_THRESHOLD / (1 + MAX_NUM_STRIKES)
//
// For the constants chosen below, this is (25%)/(1+4) ⇒ 5%.
// This may seem low, but it ensures a lower bound on
// the absolute worst-case utilization. Without this check,
// this would be theoretically 0%, which is infinitely worse.
//
// See https://go.dev/issue/27735.
switch {
case cap(e.buf) <= 4<<10: // always recycle buffers smaller than 4KiB
e.bufStats.strikes = 0
case cap(e.buf)/4 <= len(e.buf): // at least 25% utilization
e.bufStats.strikes = 0
case e.bufStats.strikes < 4: // at most 4 strikes
e.bufStats.strikes++
default: // discard the buffer; too large and too often under-utilized
e.bufStats.strikes = 0
e.bufStats.prevLen = len(e.buf) // heuristic for size to allocate next time
e.buf = nil
}
bufferedEncoderPool.Put(e)
}
func getStreamingEncoder(w io.Writer, o EncodeOptions) *Encoder {
if _, ok := w.(*bytes.Buffer); ok {
e := bytesBufferEncoderPool.Get().(*Encoder)
e.reset(nil, w, o) // buffer taken from bytes.Buffer
return e
} else {
e := streamingEncoderPool.Get().(*Encoder)
e.reset(e.buf[:0], w, o) // preserve existing buffer
return e
}
}
func putStreamingEncoder(e *Encoder) {
if _, ok := e.wr.(*bytes.Buffer); ok {
bytesBufferEncoderPool.Put(e)
} else {
if cap(e.buf) > 64<<10 {
e.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingEncoderPool.Put(e)
}
}
var (
// This does not own the internal buffer since it is externally provided.
bufferedDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON fetched from the underlying io.Reader.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferDecoderPool = bufferedDecoderPool
)
func getBufferedDecoder(b []byte, o DecodeOptions) *Decoder {
d := bufferedDecoderPool.Get().(*Decoder)
d.reset(b, nil, o)
return d
}
func putBufferedDecoder(d *Decoder) {
bufferedDecoderPool.Put(d)
}
func getStreamingDecoder(r io.Reader, o DecodeOptions) *Decoder {
if _, ok := r.(*bytes.Buffer); ok {
d := bytesBufferDecoderPool.Get().(*Decoder)
d.reset(nil, r, o) // buffer taken from bytes.Buffer
return d
} else {
d := streamingDecoderPool.Get().(*Decoder)
d.reset(d.buf[:0], r, o) // preserve existing buffer
return d
}
}
func putStreamingDecoder(d *Decoder) {
if _, ok := d.rd.(*bytes.Buffer); ok {
bytesBufferDecoderPool.Put(d)
} else {
if cap(d.buf) > 64<<10 {
d.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingDecoderPool.Put(d)
}
}

View File

@ -0,0 +1,747 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"math"
"strconv"
)
var (
errMissingName = &SyntacticError{str: "missing string for object name"}
errMissingColon = &SyntacticError{str: "missing character ':' after object name"}
errMissingValue = &SyntacticError{str: "missing value after object name"}
errMissingComma = &SyntacticError{str: "missing character ',' after object or array value"}
errMismatchDelim = &SyntacticError{str: "mismatching structural token for object or array"}
)
const errInvalidNamespace = jsonError("object namespace is in an invalid state")
type state struct {
// tokens validates whether the next token kind is valid.
tokens stateMachine
// names is a stack of object names.
// Not used if AllowDuplicateNames is true.
names objectNameStack
// namespaces is a stack of object namespaces.
// For performance reasons, Encoder or Decoder may not update this
// if Marshal or Unmarshal is able to track names in a more efficient way.
// See makeMapArshaler and makeStructArshaler.
// Not used if AllowDuplicateNames is true.
namespaces objectNamespaceStack
}
func (s *state) reset() {
s.tokens.reset()
s.names.reset()
s.namespaces.reset()
}
// appendStackPointer appends a JSON Pointer (RFC 6901) to the current value.
// The returned pointer is only accurate if s.names is populated,
// otherwise it uses the numeric index as the object member name.
//
// Invariant: Must call s.names.copyQuotedBuffer beforehand.
func (s state) appendStackPointer(b []byte) []byte {
var objectDepth int
for i := 1; i < s.tokens.depth(); i++ {
e := s.tokens.index(i)
if e.length() == 0 {
break // empty object or array
}
b = append(b, '/')
switch {
case e.isObject():
if objectDepth < s.names.length() {
for _, c := range s.names.getUnquoted(objectDepth) {
// Per RFC 6901, section 3, escape '~' and '/' characters.
switch c {
case '~':
b = append(b, "~0"...)
case '/':
b = append(b, "~1"...)
default:
b = append(b, c)
}
}
} else {
// Since the names stack is unpopulated, the name is unknown.
// As a best-effort replacement, use the numeric member index.
// While inaccurate, it produces a syntactically valid pointer.
b = strconv.AppendUint(b, uint64((e.length()-1)/2), 10)
}
objectDepth++
case e.isArray():
b = strconv.AppendUint(b, uint64(e.length()-1), 10)
}
}
return b
}
// stateMachine is a push-down automaton that validates whether
// a sequence of tokens is valid or not according to the JSON grammar.
// It is useful for both encoding and decoding.
//
// It is a stack where each entry represents a nested JSON object or array.
// The stack has a minimum depth of 1 where the first level is a
// virtual JSON array to handle a stream of top-level JSON values.
// The top-level virtual JSON array is special in that it doesn't require commas
// between each JSON value.
//
// For performance, most methods are carefully written to be inlineable.
// The zero value is a valid state machine ready for use.
type stateMachine struct {
stack []stateEntry
last stateEntry
}
// reset resets the state machine.
// The machine always starts with a minimum depth of 1.
func (m *stateMachine) reset() {
m.stack = m.stack[:0]
if cap(m.stack) > 1<<10 {
m.stack = nil
}
m.last = stateTypeArray
}
// depth is the current nested depth of JSON objects and arrays.
// It is one-indexed (i.e., top-level values have a depth of 1).
func (m stateMachine) depth() int {
return len(m.stack) + 1
}
// index returns a reference to the ith entry.
// It is only valid until the next push method call.
func (m *stateMachine) index(i int) *stateEntry {
if i == len(m.stack) {
return &m.last
}
return &m.stack[i]
}
// depthLength reports the current nested depth and
// the length of the last JSON object or array.
func (m stateMachine) depthLength() (int, int) {
return m.depth(), m.last.length()
}
// appendLiteral appends a JSON literal as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendLiteral() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
return nil
}
}
// appendString appends a JSON string as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendString() error {
switch {
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
return nil
}
}
// appendNumber appends a JSON number as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendNumber() error {
return m.appendLiteral()
}
// pushObject appends a JSON start object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushObject() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
m.stack = append(m.stack, m.last)
m.last = stateTypeObject
return nil
}
}
// popObject appends a JSON end object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popObject() error {
switch {
case !m.last.isObject():
return errMismatchDelim
case m.last.needObjectValue():
return errMissingValue
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last = m.stack[len(m.stack)-1]
m.stack = m.stack[:len(m.stack)-1]
return nil
}
}
// pushArray appends a JSON start array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushArray() error {
switch {
case m.last.needObjectName():
return errMissingName
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last.increment()
m.stack = append(m.stack, m.last)
m.last = stateTypeArray
return nil
}
}
// popArray appends a JSON end array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popArray() error {
switch {
case !m.last.isArray() || len(m.stack) == 0: // forbid popping top-level virtual JSON array
return errMismatchDelim
case !m.last.isValidNamespace():
return errInvalidNamespace
default:
m.last = m.stack[len(m.stack)-1]
m.stack = m.stack[:len(m.stack)-1]
return nil
}
}
// needIndent reports whether indent whitespace should be injected.
// A zero value means that no whitespace should be injected.
// A positive value means '\n', indentPrefix, and (n-1) copies of indentBody
// should be appended to the output immediately before the next token.
func (m stateMachine) needIndent(next Kind) (n int) {
willEnd := next == '}' || next == ']'
switch {
case m.depth() == 1:
return 0 // top-level values are never indented
case m.last.length() == 0 && willEnd:
return 0 // an empty object or array is never indented
case m.last.length() == 0 || m.last.needImplicitComma(next):
return m.depth()
case willEnd:
return m.depth() - 1
default:
return 0
}
}
// mayAppendDelim appends a colon or comma that may precede the next token.
func (m stateMachine) mayAppendDelim(b []byte, next Kind) []byte {
switch {
case m.last.needImplicitColon():
return append(b, ':')
case m.last.needImplicitComma(next) && len(m.stack) != 0: // comma not needed for top-level values
return append(b, ',')
default:
return b
}
}
// needDelim reports whether a colon or comma token should be implicitly emitted
// before the next token of the specified kind.
// A zero value means no delimiter should be emitted.
func (m stateMachine) needDelim(next Kind) (delim byte) {
switch {
case m.last.needImplicitColon():
return ':'
case m.last.needImplicitComma(next) && len(m.stack) != 0: // comma not needed for top-level values
return ','
default:
return 0
}
}
// checkDelim reports whether the specified delimiter should be there given
// the kind of the next token that appears immediately afterwards.
func (m stateMachine) checkDelim(delim byte, next Kind) error {
switch needDelim := m.needDelim(next); {
case needDelim == delim:
return nil
case needDelim == ':':
return errMissingColon
case needDelim == ',':
return errMissingComma
default:
return newInvalidCharacterError([]byte{delim}, "before next token")
}
}
// invalidateDisabledNamespaces marks all disabled namespaces as invalid.
//
// For efficiency, Marshal and Unmarshal may disable namespaces since there are
// more efficient ways to track duplicate names. However, if an error occurs,
// the namespaces in Encoder or Decoder will be left in an inconsistent state.
// Mark the namespaces as invalid so that future method calls on
// Encoder or Decoder will return an error.
func (m *stateMachine) invalidateDisabledNamespaces() {
for i := 0; i < m.depth(); i++ {
e := m.index(i)
if !e.isActiveNamespace() {
e.invalidateNamespace()
}
}
}
// stateEntry encodes several artifacts within a single unsigned integer:
// - whether this represents a JSON object or array,
// - whether this object should check for duplicate names, and
// - how many elements are in this JSON object or array.
type stateEntry uint64
const (
// The type mask (1 bit) records whether this is a JSON object or array.
stateTypeMask stateEntry = 0x8000_0000_0000_0000
stateTypeObject stateEntry = 0x8000_0000_0000_0000
stateTypeArray stateEntry = 0x0000_0000_0000_0000
// The name check mask (2 bit) records whether to update
// the namespaces for the current JSON object and
// whether the namespace is valid.
stateNamespaceMask stateEntry = 0x6000_0000_0000_0000
stateDisableNamespace stateEntry = 0x4000_0000_0000_0000
stateInvalidNamespace stateEntry = 0x2000_0000_0000_0000
// The count mask (61 bits) records the number of elements.
stateCountMask stateEntry = 0x1fff_ffff_ffff_ffff
stateCountLSBMask stateEntry = 0x0000_0000_0000_0001
stateCountOdd stateEntry = 0x0000_0000_0000_0001
stateCountEven stateEntry = 0x0000_0000_0000_0000
)
// length reports the number of elements in the JSON object or array.
// Each name and value in an object entry is treated as a separate element.
func (e stateEntry) length() int {
return int(e & stateCountMask)
}
// isObject reports whether this is a JSON object.
func (e stateEntry) isObject() bool {
return e&stateTypeMask == stateTypeObject
}
// isArray reports whether this is a JSON array.
func (e stateEntry) isArray() bool {
return e&stateTypeMask == stateTypeArray
}
// needObjectName reports whether the next token must be a JSON string,
// which is necessary for JSON object names.
func (e stateEntry) needObjectName() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountEven
}
// needImplicitColon reports whether an colon should occur next,
// which always occurs after JSON object names.
func (e stateEntry) needImplicitColon() bool {
return e.needObjectValue()
}
// needObjectValue reports whether the next token must be a JSON value,
// which is necessary after every JSON object name.
func (e stateEntry) needObjectValue() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountOdd
}
// needImplicitComma reports whether an comma should occur next,
// which always occurs after a value in a JSON object or array
// before the next value (or name).
func (e stateEntry) needImplicitComma(next Kind) bool {
return !e.needObjectValue() && e.length() > 0 && next != '}' && next != ']'
}
// increment increments the number of elements for the current object or array.
// This assumes that overflow won't practically be an issue since
// 1<<bits.OnesCount(stateCountMask) is sufficiently large.
func (e *stateEntry) increment() {
(*e)++
}
// decrement decrements the number of elements for the current object or array.
// It is the callers responsibility to ensure that e.length > 0.
func (e *stateEntry) decrement() {
(*e)--
}
// disableNamespace disables the JSON object namespace such that the
// Encoder or Decoder no longer updates the namespace.
func (e *stateEntry) disableNamespace() {
*e |= stateDisableNamespace
}
// isActiveNamespace reports whether the JSON object namespace is actively
// being updated and used for duplicate name checks.
func (e stateEntry) isActiveNamespace() bool {
return e&(stateDisableNamespace) == 0
}
// invalidateNamespace marks the JSON object namespace as being invalid.
func (e *stateEntry) invalidateNamespace() {
*e |= stateInvalidNamespace
}
// isValidNamespace reports whether the JSON object namespace is valid.
func (e stateEntry) isValidNamespace() bool {
return e&(stateInvalidNamespace) == 0
}
// objectNameStack is a stack of names when descending into a JSON object.
// In contrast to objectNamespaceStack, this only has to remember a single name
// per JSON object.
//
// This data structure may contain offsets to encodeBuffer or decodeBuffer.
// It violates clean abstraction of layers, but is significantly more efficient.
// This ensures that popping and pushing in the common case is a trivial
// push/pop of an offset integer.
//
// The zero value is an empty names stack ready for use.
type objectNameStack struct {
// offsets is a stack of offsets for each name.
// A non-negative offset is the ending offset into the local names buffer.
// A negative offset is the bit-wise inverse of a starting offset into
// a remote buffer (e.g., encodeBuffer or decodeBuffer).
// A math.MinInt offset at the end implies that the last object is empty.
// Invariant: Positive offsets always occur before negative offsets.
offsets []int
// unquotedNames is a back-to-back concatenation of names.
unquotedNames []byte
}
func (ns *objectNameStack) reset() {
ns.offsets = ns.offsets[:0]
ns.unquotedNames = ns.unquotedNames[:0]
if cap(ns.offsets) > 1<<6 {
ns.offsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.unquotedNames) > 1<<10 {
ns.unquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
func (ns *objectNameStack) length() int {
return len(ns.offsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
// It returns an empty string if the last object is empty.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) getUnquoted(i int) []byte {
ns.ensureCopiedBuffer()
if i == 0 {
return ns.unquotedNames[:ns.offsets[0]]
} else {
return ns.unquotedNames[ns.offsets[i-1]:ns.offsets[i-0]]
}
}
// invalidOffset indicates that the last JSON object currently has no name.
const invalidOffset = math.MinInt
// push descends into a nested JSON object.
func (ns *objectNameStack) push() {
ns.offsets = append(ns.offsets, invalidOffset)
}
// replaceLastQuotedOffset replaces the last name with the starting offset
// to the quoted name in some remote buffer. All offsets provided must be
// relative to the same buffer until copyQuotedBuffer is called.
func (ns *objectNameStack) replaceLastQuotedOffset(i int) {
// Use bit-wise inversion instead of naive multiplication by -1 to avoid
// ambiguity regarding zero (which is a valid offset into the names field).
// Bit-wise inversion is mathematically equivalent to -i-1,
// such that 0 becomes -1, 1 becomes -2, and so forth.
// This ensures that remote offsets are always negative.
ns.offsets[len(ns.offsets)-1] = ^i
}
// replaceLastUnquotedName replaces the last name with the provided name.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) replaceLastUnquotedName(s string) {
ns.ensureCopiedBuffer()
var startOffset int
if len(ns.offsets) > 1 {
startOffset = ns.offsets[len(ns.offsets)-2]
}
ns.unquotedNames = append(ns.unquotedNames[:startOffset], s...)
ns.offsets[len(ns.offsets)-1] = len(ns.unquotedNames)
}
// clearLast removes any name in the last JSON object.
// It is semantically equivalent to ns.push followed by ns.pop.
func (ns *objectNameStack) clearLast() {
ns.offsets[len(ns.offsets)-1] = invalidOffset
}
// pop ascends out of a nested JSON object.
func (ns *objectNameStack) pop() {
ns.offsets = ns.offsets[:len(ns.offsets)-1]
}
// copyQuotedBuffer copies names from the remote buffer into the local names
// buffer so that there are no more offset references into the remote buffer.
// This allows the remote buffer to change contents without affecting
// the names that this data structure is trying to remember.
func (ns *objectNameStack) copyQuotedBuffer(b []byte) {
// Find the first negative offset.
var i int
for i = len(ns.offsets) - 1; i >= 0 && ns.offsets[i] < 0; i-- {
continue
}
// Copy each name from the remote buffer into the local buffer.
for i = i + 1; i < len(ns.offsets); i++ {
if i == len(ns.offsets)-1 && ns.offsets[i] == invalidOffset {
if i == 0 {
ns.offsets[i] = 0
} else {
ns.offsets[i] = ns.offsets[i-1]
}
break // last JSON object had a push without any names
}
// As a form of Hyrum proofing, we write an invalid character into the
// buffer to make misuse of Decoder.ReadToken more obvious.
// We need to undo that mutation here.
quotedName := b[^ns.offsets[i]:]
if quotedName[0] == invalidateBufferByte {
quotedName[0] = '"'
}
// Append the unquoted name to the local buffer.
var startOffset int
if i > 0 {
startOffset = ns.offsets[i-1]
}
if n := consumeSimpleString(quotedName); n > 0 {
ns.unquotedNames = append(ns.unquotedNames[:startOffset], quotedName[len(`"`):n-len(`"`)]...)
} else {
ns.unquotedNames, _ = unescapeString(ns.unquotedNames[:startOffset], quotedName)
}
ns.offsets[i] = len(ns.unquotedNames)
}
}
func (ns *objectNameStack) ensureCopiedBuffer() {
if len(ns.offsets) > 0 && ns.offsets[len(ns.offsets)-1] < 0 {
panic("BUG: copyQuotedBuffer not called beforehand")
}
}
// objectNamespaceStack is a stack of object namespaces.
// This data structure assists in detecting duplicate names.
type objectNamespaceStack []objectNamespace
// reset resets the object namespace stack.
func (nss *objectNamespaceStack) reset() {
if cap(*nss) > 1<<10 {
*nss = nil
}
*nss = (*nss)[:0]
}
// push starts a new namespace for a nested JSON object.
func (nss *objectNamespaceStack) push() {
if cap(*nss) > len(*nss) {
*nss = (*nss)[:len(*nss)+1]
nss.last().reset()
} else {
*nss = append(*nss, objectNamespace{})
}
}
// last returns a pointer to the last JSON object namespace.
func (nss objectNamespaceStack) last() *objectNamespace {
return &nss[len(nss)-1]
}
// pop terminates the namespace for a nested JSON object.
func (nss *objectNamespaceStack) pop() {
*nss = (*nss)[:len(*nss)-1]
}
// objectNamespace is the namespace for a JSON object.
// In contrast to objectNameStack, this needs to remember a all names
// per JSON object.
//
// The zero value is an empty namespace ready for use.
type objectNamespace struct {
// It relies on a linear search over all the names before switching
// to use a Go map for direct lookup.
// endOffsets is a list of offsets to the end of each name in buffers.
// The length of offsets is the number of names in the namespace.
endOffsets []uint
// allUnquotedNames is a back-to-back concatenation of every name in the namespace.
allUnquotedNames []byte
// mapNames is a Go map containing every name in the namespace.
// Only valid if non-nil.
mapNames map[string]struct{}
}
// reset resets the namespace to be empty.
func (ns *objectNamespace) reset() {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
ns.mapNames = nil
if cap(ns.endOffsets) > 1<<6 {
ns.endOffsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.allUnquotedNames) > 1<<10 {
ns.allUnquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
// length reports the number of names in the namespace.
func (ns *objectNamespace) length() int {
return len(ns.endOffsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
func (ns *objectNamespace) getUnquoted(i int) []byte {
if i == 0 {
return ns.allUnquotedNames[:ns.endOffsets[0]]
} else {
return ns.allUnquotedNames[ns.endOffsets[i-1]:ns.endOffsets[i-0]]
}
}
// lastUnquoted retrieves the last name in the namespace.
func (ns *objectNamespace) lastUnquoted() []byte {
return ns.getUnquoted(ns.length() - 1)
}
// insertQuoted inserts a name and reports whether it was inserted,
// which only occurs if name is not already in the namespace.
// The provided name must be a valid JSON string.
func (ns *objectNamespace) insertQuoted(name []byte, isVerbatim bool) bool {
if isVerbatim {
name = name[len(`"`) : len(name)-len(`"`)]
}
return ns.insert(name, !isVerbatim)
}
func (ns *objectNamespace) insertUnquoted(name []byte) bool {
return ns.insert(name, false)
}
func (ns *objectNamespace) insert(name []byte, quoted bool) bool {
var allNames []byte
if quoted {
allNames, _ = unescapeString(ns.allUnquotedNames, name)
} else {
allNames = append(ns.allUnquotedNames, name...)
}
name = allNames[len(ns.allUnquotedNames):]
// Switch to a map if the buffer is too large for linear search.
// This does not add the current name to the map.
if ns.mapNames == nil && (ns.length() > 64 || len(ns.allUnquotedNames) > 1024) {
ns.mapNames = make(map[string]struct{})
var startOffset uint
for _, endOffset := range ns.endOffsets {
name := ns.allUnquotedNames[startOffset:endOffset]
ns.mapNames[string(name)] = struct{}{} // allocates a new string
startOffset = endOffset
}
}
if ns.mapNames == nil {
// Perform linear search over the buffer to find matching names.
// It provides O(n) lookup, but does not require any allocations.
var startOffset uint
for _, endOffset := range ns.endOffsets {
if string(ns.allUnquotedNames[startOffset:endOffset]) == string(name) {
return false
}
startOffset = endOffset
}
} else {
// Use the map if it is populated.
// It provides O(1) lookup, but requires a string allocation per name.
if _, ok := ns.mapNames[string(name)]; ok {
return false
}
ns.mapNames[string(name)] = struct{}{} // allocates a new string
}
ns.allUnquotedNames = allNames
ns.endOffsets = append(ns.endOffsets, uint(len(ns.allUnquotedNames)))
return true
}
// removeLast removes the last name in the namespace.
func (ns *objectNamespace) removeLast() {
if ns.mapNames != nil {
delete(ns.mapNames, string(ns.lastUnquoted()))
}
if ns.length()-1 == 0 {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
} else {
ns.endOffsets = ns.endOffsets[:ns.length()-1]
ns.allUnquotedNames = ns.allUnquotedNames[:ns.endOffsets[ns.length()-1]]
}
}
type uintSet64 uint64
func (s uintSet64) has(i uint) bool { return s&(1<<i) > 0 }
func (s *uintSet64) set(i uint) { *s |= 1 << i }
// uintSet is a set of unsigned integers.
// It is optimized for most integers being close to zero.
type uintSet struct {
lo uintSet64
hi []uintSet64
}
// has reports whether i is in the set.
func (s *uintSet) has(i uint) bool {
if i < 64 {
return s.lo.has(i)
} else {
i -= 64
iHi, iLo := int(i/64), uint(i%64)
return iHi < len(s.hi) && s.hi[iHi].has(iLo)
}
}
// insert inserts i into the set and reports whether it was the first insertion.
func (s *uintSet) insert(i uint) bool {
// TODO: Make this inlineable at least for the lower 64-bit case.
if i < 64 {
has := s.lo.has(i)
s.lo.set(i)
return !has
} else {
i -= 64
iHi, iLo := int(i/64), uint(i%64)
if iHi >= len(s.hi) {
s.hi = append(s.hi, make([]uintSet64, iHi+1-len(s.hi))...)
s.hi = s.hi[:cap(s.hi)]
}
has := s.hi[iHi].has(iLo)
s.hi[iHi].set(iLo)
return !has
}
}

View File

@ -0,0 +1,522 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"math"
"strconv"
)
// NOTE: Token is analogous to v1 json.Token.
const (
maxInt64 = math.MaxInt64
minInt64 = math.MinInt64
maxUint64 = math.MaxUint64
minUint64 = 0 // for consistency and readability purposes
invalidTokenPanic = "invalid json.Token; it has been voided by a subsequent json.Decoder call"
)
// Token represents a lexical JSON token, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a start or end delimiter for a JSON object (i.e., { or } )
// - a start or end delimiter for a JSON array (i.e., [ or ] )
//
// A Token cannot represent entire array or object values, while a RawValue can.
// There is no Token to represent commas and colons since
// these structural tokens can be inferred from the surrounding context.
type Token struct {
nonComparable
// Tokens can exist in either a "raw" or an "exact" form.
// Tokens produced by the Decoder are in the "raw" form.
// Tokens returned by constructors are usually in the "exact" form.
// The Encoder accepts Tokens in either the "raw" or "exact" form.
//
// The following chart shows the possible values for each Token type:
// ╔═════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠═════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚═════════════════╩════════════╧════════════╧════════════╝
//
// Notes:
// - For tokens stored in "raw" form, the num field contains the
// absolute offset determined by raw.previousOffsetStart().
// The buffer itself is stored in raw.previousBuffer().
// - JSON literals and structural characters are always in the "raw" form.
// - JSON strings and numbers can be in either "raw" or "exact" forms.
// - The exact zero value of JSON strings and numbers in the "exact" forms
// have ambiguous representation. Thus, they are always represented
// in the "raw" form.
// raw contains a reference to the raw decode buffer.
// If non-nil, then its value takes precedence over str and num.
// It is only valid if num == raw.previousOffsetStart().
raw *decodeBuffer
// str is the unescaped JSON string if num is zero.
// Otherwise, it is "f", "i", or "u" if num should be interpreted
// as a float64, int64, or uint64, respectively.
str string
// num is a float64, int64, or uint64 stored as a uint64 value.
// It is non-zero for any JSON number in the "exact" form.
num uint64
}
// TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
var (
Null Token = rawToken("null")
False Token = rawToken("false")
True Token = rawToken("true")
ObjectStart Token = rawToken("{")
ObjectEnd Token = rawToken("}")
ArrayStart Token = rawToken("[")
ArrayEnd Token = rawToken("]")
zeroString Token = rawToken(`""`)
zeroNumber Token = rawToken(`0`)
nanString Token = String("NaN")
pinfString Token = String("Infinity")
ninfString Token = String("-Infinity")
)
func rawToken(s string) Token {
return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
}
// Bool constructs a Token representing a JSON boolean.
func Bool(b bool) Token {
if b {
return True
}
return False
}
// String construct a Token representing a JSON string.
// The provided string should contain valid UTF-8, otherwise invalid characters
// may be mangled as the Unicode replacement character.
func String(s string) Token {
if len(s) == 0 {
return zeroString
}
return Token{str: s}
}
// Float constructs a Token representing a JSON number.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
switch {
case math.Float64bits(n) == 0:
return zeroNumber
case math.IsNaN(n):
return nanString
case math.IsInf(n, +1):
return pinfString
case math.IsInf(n, -1):
return ninfString
}
return Token{str: "f", num: math.Float64bits(n)}
}
// Int constructs a Token representing a JSON number from an int64.
func Int(n int64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "i", num: uint64(n)}
}
// Uint constructs a Token representing a JSON number from a uint64.
func Uint(n uint64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "u", num: uint64(n)}
}
// Clone makes a copy of the Token such that its value remains valid
// even after a subsequent Decoder.Read call.
func (t Token) Clone() Token {
// TODO: Allow caller to avoid any allocations?
if raw := t.raw; raw != nil {
// Avoid copying globals.
if t.raw.prevStart == 0 {
switch t.raw {
case Null.raw:
return Null
case False.raw:
return False
case True.raw:
return True
case ObjectStart.raw:
return ObjectStart
case ObjectEnd.raw:
return ObjectEnd
case ArrayStart.raw:
return ArrayStart
case ArrayEnd.raw:
return ArrayEnd
}
}
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
// TODO(https://go.dev/issue/45038): Use bytes.Clone.
buf := append([]byte(nil), raw.previousBuffer()...)
return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
}
return t
}
// Bool returns the value for a JSON boolean.
// It panics if the token kind is not a JSON boolean.
func (t Token) Bool() bool {
switch t.raw {
case True.raw:
return true
case False.raw:
return false
default:
panic("invalid JSON token kind: " + t.Kind().String())
}
}
// appendString appends a JSON string to dst and returns it.
// It panics if t is not a JSON string.
func (t Token) appendString(dst []byte, validateUTF8, preserveRaw bool, escapeRune func(rune) bool) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw string value.
buf := raw.previousBuffer()
if Kind(buf[0]) == '"' {
if escapeRune == nil && consumeSimpleString(buf) == len(buf) {
return append(dst, buf...), nil
}
dst, _, err := reformatString(dst, buf, validateUTF8, preserveRaw, escapeRune)
return dst, err
}
} else if len(t.str) != 0 && t.num == 0 {
// Handle exact string value.
return appendString(dst, t.str, validateUTF8, escapeRune)
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// String returns the unescaped string value for a JSON string.
// For other JSON kinds, this returns the raw JSON represention.
func (t Token) String() string {
// This is inlinable to take advantage of "function outlining".
// This avoids an allocation for the string(b) conversion
// if the caller does not use the string in an escaping manner.
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
s, b := t.string()
if len(b) > 0 {
return string(b)
}
return s
}
func (t Token) string() (string, []byte) {
if raw := t.raw; raw != nil {
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if buf[0] == '"' {
// TODO: Preserve valueFlags in Token?
isVerbatim := consumeSimpleString(buf) == len(buf)
return "", unescapeStringMayCopy(buf, isVerbatim)
}
// Handle tokens that are not JSON strings for fmt.Stringer.
return "", buf
}
if len(t.str) != 0 && t.num == 0 {
return t.str, nil
}
// Handle tokens that are not JSON strings for fmt.Stringer.
if t.num > 0 {
switch t.str[0] {
case 'f':
return string(appendNumber(nil, math.Float64frombits(t.num), 64)), nil
case 'i':
return strconv.FormatInt(int64(t.num), 10), nil
case 'u':
return strconv.FormatUint(uint64(t.num), 10), nil
}
}
return "<invalid json.Token>", nil
}
// appendNumber appends a JSON number to dst and returns it.
// It panics if t is not a JSON number.
func (t Token) appendNumber(dst []byte, canonicalize bool) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw number value.
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
if !canonicalize {
return append(dst, buf...), nil
}
dst, _, err := reformatNumber(dst, buf, canonicalize)
return dst, err
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return appendNumber(dst, math.Float64frombits(t.num), 64), nil
case 'i':
return strconv.AppendInt(dst, int64(t.num), 10), nil
case 'u':
return strconv.AppendUint(dst, uint64(t.num), 10), nil
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Float returns the floating-point value for a JSON number.
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
// It panics for all other cases.
func (t Token) Float() float64 {
if raw := t.raw; raw != nil {
// Handle raw number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
fv, _ := parseFloat(buf, 64)
return fv
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return math.Float64frombits(t.num)
case 'i':
return float64(int64(t.num))
case 'u':
return float64(uint64(t.num))
}
}
// Handle string values with "NaN", "Infinity", or "-Infinity".
if t.Kind() == '"' {
switch t.String() {
case "NaN":
return math.NaN()
case "Infinity":
return math.Inf(+1)
case "-Infinity":
return math.Inf(-1)
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Int returns the signed integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an int64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Int() int64 {
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.previousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if numAbs, ok := parseDecUint(buf); ok {
if neg {
if numAbs > -minInt64 {
return minInt64
}
return -1 * int64(numAbs)
} else {
if numAbs > +maxInt64 {
return maxInt64
}
return +1 * int64(numAbs)
}
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'i':
return int64(t.num)
case 'u':
if uint64(t.num) > maxInt64 {
return maxInt64
}
return int64(uint64(t.num))
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxInt64:
return maxInt64
case fv <= minInt64:
return minInt64
default:
return int64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Uint returns the unsigned integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an uint64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Uint() uint64 {
// NOTE: This accessor returns 0 for any negative JSON number,
// which might be surprising, but is at least consistent with the behavior
// of saturating out-of-bounds numbers to the closest representable number.
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.previousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if num, ok := parseDecUint(buf); ok {
if neg {
return minUint64
}
return num
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'u':
return uint64(t.num)
case 'i':
if int64(t.num) < minUint64 {
return minUint64
}
return uint64(int64(t.num))
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxUint64:
return maxUint64
case fv <= minUint64:
return minUint64
default:
return uint64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Kind returns the token kind.
func (t Token) Kind() Kind {
switch {
case t.raw != nil:
raw := t.raw
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
return Kind(t.raw.buf[raw.prevStart]).normalize()
case t.num != 0:
return '0'
case len(t.str) != 0:
return '"'
default:
return invalidKind
}
}
// Kind represents each possible JSON token kind with a single byte,
// which is conveniently the first byte of that kind's grammar
// with the restriction that numbers always be represented with '0':
//
// - 'n': null
// - 'f': false
// - 't': true
// - '"': string
// - '0': number
// - '{': object start
// - '}': object end
// - '[': array start
// - ']': array end
//
// An invalid kind is usually represented using 0,
// but may be non-zero due to invalid JSON data.
type Kind byte
const invalidKind Kind = 0
// String prints the kind in a humanly readable fashion.
func (k Kind) String() string {
switch k {
case 'n':
return "null"
case 'f':
return "false"
case 't':
return "true"
case '"':
return "string"
case '0':
return "number"
case '{':
return "{"
case '}':
return "}"
case '[':
return "["
case ']':
return "]"
default:
return "<invalid json.Kind: " + quoteRune([]byte{byte(k)}) + ">"
}
}
// normalize coalesces all possible starting characters of a number as just '0'.
func (k Kind) normalize() Kind {
if k == '-' || ('0' <= k && k <= '9') {
return '0'
}
return k
}

View File

@ -0,0 +1,375 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
"bytes"
"errors"
"io"
"sort"
"sync"
"unicode/utf16"
"unicode/utf8"
)
// NOTE: RawValue is analogous to v1 json.RawMessage.
// RawValue represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - an entire JSON object (e.g., {"fizz":"buzz"} )
// - an entire JSON array (e.g., [1,2,3] )
//
// RawValue can represent entire array or object values, while Token cannot.
// RawValue may contain leading and/or trailing whitespace.
type RawValue []byte
// Clone returns a copy of v.
func (v RawValue) Clone() RawValue {
if v == nil {
return nil
}
return append(RawValue{}, v...)
}
// String returns the string formatting of v.
func (v RawValue) String() string {
if v == nil {
return "null"
}
return string(v)
}
// IsValid reports whether the raw JSON value is syntactically valid
// according to RFC 7493.
//
// It verifies whether the input is properly encoded as UTF-8,
// that escape sequences within strings decode to valid Unicode codepoints, and
// that all names in each object are unique.
// It does not verify whether numbers are representable within the limits
// of any common numeric type (e.g., float64, int64, or uint64).
func (v RawValue) IsValid() bool {
d := getBufferedDecoder(v, DecodeOptions{})
defer putBufferedDecoder(d)
_, errVal := d.ReadValue()
_, errEOF := d.ReadToken()
return errVal == nil && errEOF == io.EOF
}
// Compact removes all whitespace from the raw JSON value.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already compacted, then the buffer is not mutated.
func (v *RawValue) Compact() error {
return v.reformat(false, false, "", "")
}
// Indent reformats the whitespace in the raw JSON value so that each element
// in a JSON object or array begins on a new, indented line beginning with
// prefix followed by one or more copies of indent according to the nesting.
// The value does not begin with the prefix nor any indention,
// to make it easier to embed inside other formatted JSON data.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already indented properly, then the buffer is not mutated.
func (v *RawValue) Indent(prefix, indent string) error {
return v.reformat(false, true, prefix, indent)
}
// Canonicalize canonicalizes the raw JSON value according to the
// JSON Canonicalization Scheme (JCS) as defined by RFC 8785
// where it produces a stable representation of a JSON value.
//
// The output stability is dependent on the stability of the application data
// (see RFC 8785, Appendix E). It cannot produce stable output from
// fundamentally unstable input. For example, if the JSON value
// contains ephemeral data (e.g., a frequently changing timestamp),
// then the value is still unstable regardless of whether this is called.
//
// Note that JCS treats all JSON numbers as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example, integer values
// beyond ±2⁵³ will lose their precision. It is recommended that
// int64 and uint64 data types be represented as a JSON string.
//
// It is guaranteed to succeed if the input is valid.
// If the value is already canonicalized, then the buffer is not mutated.
func (v *RawValue) Canonicalize() error {
return v.reformat(true, false, "", "")
}
// TODO: Instead of implementing the v1 Marshaler/Unmarshaler,
// consider implementing the v2 versions instead.
// MarshalJSON returns v as the JSON encoding of v.
// It returns the stored value as the raw JSON output without any validation.
// If v is nil, then this returns a JSON null.
func (v RawValue) MarshalJSON() ([]byte, error) {
// NOTE: This matches the behavior of v1 json.RawMessage.MarshalJSON.
if v == nil {
return []byte("null"), nil
}
return v, nil
}
// UnmarshalJSON sets v as the JSON encoding of b.
// It stores a copy of the provided raw JSON input without any validation.
func (v *RawValue) UnmarshalJSON(b []byte) error {
// NOTE: This matches the behavior of v1 json.RawMessage.UnmarshalJSON.
if v == nil {
return errors.New("json.RawValue: UnmarshalJSON on nil pointer")
}
*v = append((*v)[:0], b...)
return nil
}
// Kind returns the starting token kind.
// For a valid value, this will never include '}' or ']'.
func (v RawValue) Kind() Kind {
if v := v[consumeWhitespace(v):]; len(v) > 0 {
return Kind(v[0]).normalize()
}
return invalidKind
}
func (v *RawValue) reformat(canonical, multiline bool, prefix, indent string) error {
var eo EncodeOptions
if canonical {
eo.AllowInvalidUTF8 = false // per RFC 8785, section 3.2.4
eo.AllowDuplicateNames = false // per RFC 8785, section 3.1
eo.canonicalizeNumbers = true // per RFC 8785, section 3.2.2.3
eo.EscapeRune = nil // per RFC 8785, section 3.2.2.2
eo.multiline = false // per RFC 8785, section 3.2.1
} else {
if s := trimLeftSpaceTab(prefix); len(s) > 0 {
panic("json: invalid character " + quoteRune([]byte(s)) + " in indent prefix")
}
if s := trimLeftSpaceTab(indent); len(s) > 0 {
panic("json: invalid character " + quoteRune([]byte(s)) + " in indent")
}
eo.AllowInvalidUTF8 = true
eo.AllowDuplicateNames = true
eo.preserveRawStrings = true
eo.multiline = multiline // in case indent is empty
eo.IndentPrefix = prefix
eo.Indent = indent
}
eo.omitTopLevelNewline = true
// Write the entire value to reformat all tokens and whitespace.
e := getBufferedEncoder(eo)
defer putBufferedEncoder(e)
if err := e.WriteValue(*v); err != nil {
return err
}
// For canonical output, we may need to reorder object members.
if canonical {
// Obtain a buffered encoder just to use its internal buffer as
// a scratch buffer in reorderObjects for reordering object members.
e2 := getBufferedEncoder(EncodeOptions{})
defer putBufferedEncoder(e2)
// Disable redundant checks performed earlier during encoding.
d := getBufferedDecoder(e.buf, DecodeOptions{AllowInvalidUTF8: true, AllowDuplicateNames: true})
defer putBufferedDecoder(d)
reorderObjects(d, &e2.buf) // per RFC 8785, section 3.2.3
}
// Store the result back into the value if different.
if !bytes.Equal(*v, e.buf) {
*v = append((*v)[:0], e.buf...)
}
return nil
}
func trimLeftSpaceTab(s string) string {
for i, r := range s {
switch r {
case ' ', '\t':
default:
return s[i:]
}
}
return ""
}
type memberName struct {
// name is the unescaped name.
name []byte
// before and after are byte offsets into Decoder.buf that represents
// the entire name/value pair. It may contain leading commas.
before, after int64
}
var memberNamePool = sync.Pool{New: func() any { return new(memberNames) }}
func getMemberNames() *memberNames {
ns := memberNamePool.Get().(*memberNames)
*ns = (*ns)[:0]
return ns
}
func putMemberNames(ns *memberNames) {
if cap(*ns) < 1<<10 {
for i := range *ns {
(*ns)[i] = memberName{} // avoid pinning name
}
memberNamePool.Put(ns)
}
}
type memberNames []memberName
func (m *memberNames) Len() int { return len(*m) }
func (m *memberNames) Less(i, j int) bool { return lessUTF16((*m)[i].name, (*m)[j].name) }
func (m *memberNames) Swap(i, j int) { (*m)[i], (*m)[j] = (*m)[j], (*m)[i] }
// reorderObjects recursively reorders all object members in place
// according to the ordering specified in RFC 8785, section 3.2.3.
//
// Pre-conditions:
// - The value is valid (i.e., no decoder errors should ever occur).
// - The value is compact (i.e., no whitespace is present).
// - Initial call is provided a Decoder reading from the start of v.
//
// Post-conditions:
// - Exactly one JSON value is read from the Decoder.
// - All fully-parsed JSON objects are reordered by directly moving
// the members in the value buffer.
//
// The runtime is approximately O(n·log(n)) + O(m·log(m)),
// where n is len(v) and m is the total number of object members.
func reorderObjects(d *Decoder, scratch *[]byte) {
switch tok, _ := d.ReadToken(); tok.Kind() {
case '{':
// Iterate and collect the name and offsets for every object member.
members := getMemberNames()
defer putMemberNames(members)
var prevName []byte
isSorted := true
beforeBody := d.InputOffset() // offset after '{'
for d.PeekKind() != '}' {
beforeName := d.InputOffset()
var flags valueFlags
name, _ := d.readValue(&flags)
name = unescapeStringMayCopy(name, flags.isVerbatim())
reorderObjects(d, scratch)
afterValue := d.InputOffset()
if isSorted && len(*members) > 0 {
isSorted = lessUTF16(prevName, name)
}
*members = append(*members, memberName{name, beforeName, afterValue})
prevName = name
}
afterBody := d.InputOffset() // offset before '}'
d.ReadToken()
// Sort the members; return early if it's already sorted.
if isSorted {
return
}
// TODO(https://go.dev/issue/47619): Use slices.Sort.
sort.Sort(members)
// Append the reordered members to a new buffer,
// then copy the reordered members back over the original members.
// Avoid swapping in place since each member may be a different size
// where moving a member over a smaller member may corrupt the data
// for subsequent members before they have been moved.
//
// The following invariant must hold:
// sum([m.after-m.before for m in members]) == afterBody-beforeBody
sorted := (*scratch)[:0]
for i, member := range *members {
if d.buf[member.before] == ',' {
member.before++ // trim leading comma
}
sorted = append(sorted, d.buf[member.before:member.after]...)
if i < len(*members)-1 {
sorted = append(sorted, ',') // append trailing comma
}
}
if int(afterBody-beforeBody) != len(sorted) {
panic("BUG: length invariant violated")
}
copy(d.buf[beforeBody:afterBody], sorted)
// Update scratch buffer to the largest amount ever used.
if len(sorted) > len(*scratch) {
*scratch = sorted
}
case '[':
for d.PeekKind() != ']' {
reorderObjects(d, scratch)
}
d.ReadToken()
}
}
// lessUTF16 reports whether x is lexicographically less than y according
// to the UTF-16 codepoints of the UTF-8 encoded input strings.
// This implements the ordering specified in RFC 8785, section 3.2.3.
// The inputs must be valid UTF-8, otherwise this may panic.
func lessUTF16(x, y []byte) bool {
// NOTE: This is an optimized, allocation-free implementation
// of lessUTF16Simple in fuzz_test.go. FuzzLessUTF16 verifies that the
// two implementations agree on the result of comparing any two strings.
isUTF16Self := func(r rune) bool {
return ('\u0000' <= r && r <= '\uD7FF') || ('\uE000' <= r && r <= '\uFFFF')
}
for {
if len(x) == 0 || len(y) == 0 {
return len(x) < len(y)
}
// ASCII fast-path.
if x[0] < utf8.RuneSelf || y[0] < utf8.RuneSelf {
if x[0] != y[0] {
return x[0] < y[0]
}
x, y = x[1:], y[1:]
continue
}
// Decode next pair of runes as UTF-8.
rx, nx := utf8.DecodeRune(x)
ry, ny := utf8.DecodeRune(y)
switch {
// Both runes encode as either a single or surrogate pair
// of UTF-16 codepoints.
case isUTF16Self(rx) == isUTF16Self(ry):
if rx != ry {
return rx < ry
}
// The x rune is a single UTF-16 codepoint, while
// the y rune is a surrogate pair of UTF-16 codepoints.
case isUTF16Self(rx):
ry, _ := utf16.EncodeRune(ry)
if rx != ry {
return rx < ry
}
panic("BUG: invalid UTF-8") // implies rx is an unpaired surrogate half
// The y rune is a single UTF-16 codepoint, while
// the x rune is a surrogate pair of UTF-16 codepoints.
case isUTF16Self(ry):
rx, _ := utf16.EncodeRune(rx)
if rx != ry {
return rx < ry
}
panic("BUG: invalid UTF-8") // implies ry is an unpaired surrogate half
}
x, y = x[nx:], y[ny:]
}
}

502
vendor/k8s.io/kube-openapi/pkg/validation/spec/fuzz.go generated vendored Normal file
View File

@ -0,0 +1,502 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package spec
import (
"github.com/go-openapi/jsonreference"
"github.com/google/go-cmp/cmp"
fuzz "github.com/google/gofuzz"
)
var SwaggerFuzzFuncs []interface{} = []interface{}{
func(v *Responses, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v.Default != nil {
// Check if we hit maxDepth and left an incomplete value
if v.Default.Description == "" {
v.Default = nil
v.StatusCodeResponses = nil
}
}
// conversion has no way to discern empty statusCodeResponses from
// nil, since "default" is always included in the map.
// So avoid empty responses list
if len(v.StatusCodeResponses) == 0 {
v.StatusCodeResponses = nil
}
},
func(v *Operation, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v != nil {
// force non-nil
v.Responses = &Responses{}
c.Fuzz(v.Responses)
v.Schemes = nil
if c.RandBool() {
v.Schemes = append(v.Schemes, "http")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "https")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "ws")
}
if c.RandBool() {
v.Schemes = append(v.Schemes, "wss")
}
// Gnostic unconditionally makes security values non-null
// So do not fuzz null values into the array.
for i, val := range v.Security {
if val == nil {
v.Security[i] = make(map[string][]string)
}
for k, v := range val {
if v == nil {
val[k] = make([]string, 0)
}
}
}
}
},
func(v map[int]Response, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
// Prevent negative numbers
num := c.Intn(4)
for i := 0; i < num+2; i++ {
val := Response{}
c.Fuzz(&val)
val.Description = c.RandString() + "x"
v[100*(i+1)+c.Intn(100)] = val
}
},
func(v map[string]PathItem, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
num := c.Intn(5)
for i := 0; i < num+2; i++ {
val := PathItem{}
c.Fuzz(&val)
// Ref params are only allowed in certain locations, so
// possibly add a few to PathItems
numRefsToAdd := c.Intn(5)
for i := 0; i < numRefsToAdd; i++ {
theRef := Parameter{}
c.Fuzz(&theRef.Refable)
val.Parameters = append(val.Parameters, theRef)
}
v["/"+c.RandString()] = val
}
},
func(v *SchemaOrArray, c fuzz.Continue) {
*v = SchemaOrArray{}
// gnostic parser just doesn't support more
// than one Schema here
v.Schema = &Schema{}
c.Fuzz(&v.Schema)
},
func(v *SchemaOrBool, c fuzz.Continue) {
*v = SchemaOrBool{}
if c.RandBool() {
v.Allows = c.RandBool()
} else {
v.Schema = &Schema{}
v.Allows = true
c.Fuzz(&v.Schema)
}
},
func(v map[string]Response, c fuzz.Continue) {
n := 0
c.Fuzz(&n)
if n == 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
return
}
// Response definitions are not allowed to
// be refs
for i := 0; i < c.Intn(5)+1; i++ {
resp := &Response{}
c.Fuzz(resp)
resp.Ref = Ref{}
resp.Description = c.RandString() + "x"
// Response refs are not vendor extensible by gnostic
resp.VendorExtensible.Extensions = nil
v[c.RandString()+"x"] = *resp
}
},
func(v *Header, c fuzz.Continue) {
if v != nil {
c.FuzzNoCustom(v)
// descendant Items of Header may not be refs
cur := v.Items
for cur != nil {
cur.Ref = Ref{}
cur = cur.Items
}
}
},
func(v *Ref, c fuzz.Continue) {
*v = Ref{}
v.Ref, _ = jsonreference.New("http://asd.com/" + c.RandString())
},
func(v *Response, c fuzz.Continue) {
*v = Response{}
if c.RandBool() {
v.Ref = Ref{}
v.Ref.Ref, _ = jsonreference.New("http://asd.com/" + c.RandString())
} else {
c.Fuzz(&v.VendorExtensible)
c.Fuzz(&v.Schema)
c.Fuzz(&v.ResponseProps)
v.Headers = nil
v.Ref = Ref{}
n := 0
c.Fuzz(&n)
if n != 0 {
// Test that fuzzer is not at maxDepth so we do not
// end up with empty elements
num := c.Intn(4)
for i := 0; i < num; i++ {
if v.Headers == nil {
v.Headers = make(map[string]Header)
}
hdr := Header{}
c.Fuzz(&hdr)
if hdr.Type == "" {
// hit maxDepth, just abort trying to make haders
v.Headers = nil
break
}
v.Headers[c.RandString()+"x"] = hdr
}
} else {
v.Headers = nil
}
}
v.Description = c.RandString() + "x"
// Gnostic parses empty as nil, so to keep avoid putting empty
if len(v.Headers) == 0 {
v.Headers = nil
}
},
func(v **Info, c fuzz.Continue) {
// Info is never nil
*v = &Info{}
c.FuzzNoCustom(*v)
(*v).Title = c.RandString() + "x"
},
func(v *Extensions, c fuzz.Continue) {
// gnostic parser only picks up x- vendor extensions
numChildren := c.Intn(5)
for i := 0; i < numChildren; i++ {
if *v == nil {
*v = Extensions{}
}
(*v)["x-"+c.RandString()] = c.RandString()
}
},
func(v *Swagger, c fuzz.Continue) {
c.FuzzNoCustom(v)
if v.Paths == nil {
// Force paths non-nil since it does not have omitempty in json tag.
// This means a perfect roundtrip (via json) is impossible,
// since we can't tell the difference between empty/unspecified paths
v.Paths = &Paths{}
c.Fuzz(v.Paths)
}
v.Swagger = "2.0"
// Gnostic support serializing ID at all
// unavoidable data loss
v.ID = ""
v.Schemes = nil
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "http")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "https")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "ws")
}
if c.RandUint64()%2 == 1 {
v.Schemes = append(v.Schemes, "wss")
}
// Gnostic unconditionally makes security values non-null
// So do not fuzz null values into the array.
for i, val := range v.Security {
if val == nil {
v.Security[i] = make(map[string][]string)
}
for k, v := range val {
if v == nil {
val[k] = make([]string, 0)
}
}
}
},
func(v *SecurityScheme, c fuzz.Continue) {
v.Description = c.RandString() + "x"
c.Fuzz(&v.VendorExtensible)
switch c.Intn(3) {
case 0:
v.Type = "basic"
case 1:
v.Type = "apiKey"
switch c.Intn(2) {
case 0:
v.In = "header"
case 1:
v.In = "query"
default:
panic("unreachable")
}
v.Name = "x" + c.RandString()
case 2:
v.Type = "oauth2"
switch c.Intn(4) {
case 0:
v.Flow = "accessCode"
v.TokenURL = "https://" + c.RandString()
v.AuthorizationURL = "https://" + c.RandString()
case 1:
v.Flow = "application"
v.TokenURL = "https://" + c.RandString()
case 2:
v.Flow = "implicit"
v.AuthorizationURL = "https://" + c.RandString()
case 3:
v.Flow = "password"
v.TokenURL = "https://" + c.RandString()
default:
panic("unreachable")
}
c.Fuzz(&v.Scopes)
default:
panic("unreachable")
}
},
func(v *interface{}, c fuzz.Continue) {
*v = c.RandString() + "x"
},
func(v *string, c fuzz.Continue) {
*v = c.RandString() + "x"
},
func(v *ExternalDocumentation, c fuzz.Continue) {
v.Description = c.RandString() + "x"
v.URL = c.RandString() + "x"
},
func(v *SimpleSchema, c fuzz.Continue) {
c.FuzzNoCustom(v)
switch c.Intn(5) {
case 0:
v.Type = "string"
case 1:
v.Type = "number"
case 2:
v.Type = "boolean"
case 3:
v.Type = "integer"
case 4:
v.Type = "array"
default:
panic("unreachable")
}
switch c.Intn(5) {
case 0:
v.CollectionFormat = "csv"
case 1:
v.CollectionFormat = "ssv"
case 2:
v.CollectionFormat = "tsv"
case 3:
v.CollectionFormat = "pipes"
case 4:
v.CollectionFormat = ""
default:
panic("unreachable")
}
// None of the types which include SimpleSchema in our definitions
// actually support "example" in the official spec
v.Example = nil
// unsupported by openapi
v.Nullable = false
},
func(v *int64, c fuzz.Continue) {
c.Fuzz(v)
// Gnostic does not differentiate between 0 and non-specified
// so avoid using 0 for fuzzer
if *v == 0 {
*v = 1
}
},
func(v *float64, c fuzz.Continue) {
c.Fuzz(v)
// Gnostic does not differentiate between 0 and non-specified
// so avoid using 0 for fuzzer
if *v == 0.0 {
*v = 1.0
}
},
func(v *Parameter, c fuzz.Continue) {
if v == nil {
return
}
c.Fuzz(&v.VendorExtensible)
if c.RandBool() {
// body param
v.Description = c.RandString() + "x"
v.Name = c.RandString() + "x"
v.In = "body"
c.Fuzz(&v.Description)
c.Fuzz(&v.Required)
v.Schema = &Schema{}
c.Fuzz(&v.Schema)
} else {
c.Fuzz(&v.SimpleSchema)
c.Fuzz(&v.CommonValidations)
v.AllowEmptyValue = false
v.Description = c.RandString() + "x"
v.Name = c.RandString() + "x"
switch c.Intn(4) {
case 0:
// Header param
v.In = "header"
case 1:
// Form data param
v.In = "formData"
v.AllowEmptyValue = c.RandBool()
case 2:
// Query param
v.In = "query"
v.AllowEmptyValue = c.RandBool()
case 3:
// Path param
v.In = "path"
v.Required = true
default:
panic("unreachable")
}
// descendant Items of Parameter may not be refs
cur := v.Items
for cur != nil {
cur.Ref = Ref{}
cur = cur.Items
}
}
},
func(v *Schema, c fuzz.Continue) {
if c.RandBool() {
// file schema
c.Fuzz(&v.Default)
c.Fuzz(&v.Description)
c.Fuzz(&v.Example)
c.Fuzz(&v.ExternalDocs)
c.Fuzz(&v.Format)
c.Fuzz(&v.ReadOnly)
c.Fuzz(&v.Required)
c.Fuzz(&v.Title)
v.Type = StringOrArray{"file"}
} else {
// normal schema
c.Fuzz(&v.SchemaProps)
c.Fuzz(&v.SwaggerSchemaProps)
c.Fuzz(&v.VendorExtensible)
// c.Fuzz(&v.ExtraProps)
// ExtraProps will not roundtrip - gnostic throws out
// unrecognized keys
}
// Not supported by official openapi v2 spec
// and stripped by k8s apiserver
v.ID = ""
v.AnyOf = nil
v.OneOf = nil
v.Not = nil
v.Nullable = false
v.AdditionalItems = nil
v.Schema = ""
v.PatternProperties = nil
v.Definitions = nil
v.Dependencies = nil
},
}
var SwaggerDiffOptions = []cmp.Option{
// cmp.Diff panics on Ref since jsonreference.Ref uses unexported fields
cmp.Comparer(func(a Ref, b Ref) bool {
return a.String() == b.String()
}),
}

View File

@ -219,8 +219,8 @@ func (k *Ref) FromGnostic(g string) error {
// Caveats:
//
// - gnostic v2 documents treats zero as unspecified for numerical fields of
//CommonValidations fields such as Maximum, Minimum, MaximumItems, etc.
//There will always be data loss if one of the values of these fields is set to zero.
// CommonValidations fields such as Maximum, Minimum, MaximumItems, etc.
// There will always be data loss if one of the values of these fields is set to zero.
//
// Returns:
//
@ -1263,6 +1263,8 @@ func (k *Schema) FromGnostic(g *openapi_v2.Schema) (ok bool, err error) {
k.AdditionalProperties.Allows = g.AdditionalProperties.GetBoolean()
} else {
k.AdditionalProperties.Schema = &Schema{}
k.AdditionalProperties.Allows = true
if nok, err := k.AdditionalProperties.Schema.FromGnostic(g.AdditionalProperties.GetSchema()); err != nil {
return false, err
} else if !nok {

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
const (
@ -62,6 +64,10 @@ func (h Header) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals this header from JSON
func (h *Header) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, h)
}
if err := json.Unmarshal(data, &h.CommonValidations); err != nil {
return err
}
@ -73,3 +79,27 @@ func (h *Header) UnmarshalJSON(data []byte) error {
}
return json.Unmarshal(data, &h.HeaderProps)
}
func (h *Header) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
HeaderProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
h.CommonValidations = x.CommonValidations
h.SimpleSchema = x.SimpleSchema
h.Extensions = x.Extensions
h.HeaderProps = x.HeaderProps
h.Extensions.sanitize()
if len(h.Extensions) == 0 {
h.Extensions = nil
}
return nil
}

View File

@ -19,6 +19,8 @@ import (
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Extensions vendor specific extensions
@ -87,6 +89,31 @@ func (e Extensions) GetObject(key string, out interface{}) error {
return nil
}
func (e Extensions) sanitize() {
for k := range e {
if !isExtensionKey(k) {
delete(e, k)
}
}
}
func (e Extensions) sanitizeWithExtra() (extra map[string]any) {
for k, v := range e {
if !isExtensionKey(k) {
if extra == nil {
extra = make(map[string]any)
}
extra[k] = v
delete(e, k)
}
}
return extra
}
func isExtensionKey(k string) bool {
return len(k) > 1 && (k[0] == 'x' || k[0] == 'X') && k[1] == '-'
}
// VendorExtensible composition block.
type VendorExtensible struct {
Extensions Extensions
@ -167,8 +194,29 @@ func (i Info) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (i *Info) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, i)
}
if err := json.Unmarshal(data, &i.InfoProps); err != nil {
return err
}
return json.Unmarshal(data, &i.VendorExtensible)
}
func (i *Info) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
InfoProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
i.VendorExtensible.Extensions = x.Extensions
i.InfoProps = x.InfoProps
return nil
}

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
const (
@ -64,6 +66,10 @@ type Items struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (i *Items) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, i)
}
var validations CommonValidations
if err := json.Unmarshal(data, &validations); err != nil {
return err
@ -87,6 +93,28 @@ func (i *Items) UnmarshalJSON(data []byte) error {
return nil
}
func (i *Items) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := i.Refable.Ref.fromMap(x.Extensions); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
i.CommonValidations = x.CommonValidations
i.SimpleSchema = x.SimpleSchema
i.VendorExtensible.Extensions = x.Extensions
return nil
}
// MarshalJSON converts this items object to JSON
func (i Items) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(i.CommonValidations)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// OperationProps describes an operation
@ -75,12 +77,34 @@ type Operation struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (o *Operation) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, o)
}
if err := json.Unmarshal(data, &o.OperationProps); err != nil {
return err
}
return json.Unmarshal(data, &o.VendorExtensible)
}
func (o *Operation) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
type OperationPropsNoMethods OperationProps // strip MarshalJSON method
var x struct {
Extensions
OperationPropsNoMethods
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
o.VendorExtensible.Extensions = x.Extensions
o.OperationProps = OperationProps(x.OperationPropsNoMethods)
return nil
}
// MarshalJSON converts this items object to JSON
func (o Operation) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(o.OperationProps)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// ParamProps describes the specific attributes of an operation parameter
@ -38,26 +40,31 @@ type ParamProps struct {
//
// There are five possible parameter types.
// * Path - Used together with [Path Templating](#pathTemplating), where the parameter value is actually part
// of the operation's URL. This does not include the host or base path of the API. For example, in `/items/{itemId}`,
// the path parameter is `itemId`.
//
// of the operation's URL. This does not include the host or base path of the API. For example, in `/items/{itemId}`,
// the path parameter is `itemId`.
//
// * Query - Parameters that are appended to the URL. For example, in `/items?id=###`, the query parameter is `id`.
// * Header - Custom headers that are expected as part of the request.
// * Body - The payload that's appended to the HTTP request. Since there can only be one payload, there can only be
// _one_ body parameter. The name of the body parameter has no effect on the parameter itself and is used for
// documentation purposes only. Since Form parameters are also in the payload, body and form parameters cannot exist
// together for the same operation.
//
// _one_ body parameter. The name of the body parameter has no effect on the parameter itself and is used for
// documentation purposes only. Since Form parameters are also in the payload, body and form parameters cannot exist
// together for the same operation.
//
// * Form - Used to describe the payload of an HTTP request when either `application/x-www-form-urlencoded` or
// `multipart/form-data` are used as the content type of the request (in Swagger's definition,
// the [`consumes`](#operationConsumes) property of an operation). This is the only parameter type that can be used
// to send files, thus supporting the `file` type. Since form parameters are sent in the payload, they cannot be
// declared together with a body parameter for the same operation. Form parameters have a different format based on
// the content-type used (for further details, consult http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4).
// * `application/x-www-form-urlencoded` - Similar to the format of Query parameters but as a payload.
// For example, `foo=1&bar=swagger` - both `foo` and `bar` are form parameters. This is normally used for simple
// parameters that are being transferred.
// * `multipart/form-data` - each parameter takes a section in the payload with an internal header.
// For example, for the header `Content-Disposition: form-data; name="submit-name"` the name of the parameter is
// `submit-name`. This type of form parameters is more commonly used for file transfers.
//
// `multipart/form-data` are used as the content type of the request (in Swagger's definition,
// the [`consumes`](#operationConsumes) property of an operation). This is the only parameter type that can be used
// to send files, thus supporting the `file` type. Since form parameters are sent in the payload, they cannot be
// declared together with a body parameter for the same operation. Form parameters have a different format based on
// the content-type used (for further details, consult http://www.w3.org/TR/html401/interact/forms.html#h-17.13.4).
// * `application/x-www-form-urlencoded` - Similar to the format of Query parameters but as a payload.
// For example, `foo=1&bar=swagger` - both `foo` and `bar` are form parameters. This is normally used for simple
// parameters that are being transferred.
// * `multipart/form-data` - each parameter takes a section in the payload with an internal header.
// For example, for the header `Content-Disposition: form-data; name="submit-name"` the name of the parameter is
// `submit-name`. This type of form parameters is more commonly used for file transfers.
//
// For more information: http://goo.gl/8us55a#parameterObject
type Parameter struct {
@ -70,6 +77,10 @@ type Parameter struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *Parameter) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
if err := json.Unmarshal(data, &p.CommonValidations); err != nil {
return err
}
@ -85,6 +96,30 @@ func (p *Parameter) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, &p.ParamProps)
}
func (p *Parameter) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
CommonValidations
SimpleSchema
Extensions
ParamProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := p.Refable.Ref.fromMap(x.Extensions); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
p.CommonValidations = x.CommonValidations
p.SimpleSchema = x.SimpleSchema
p.VendorExtensible.Extensions = x.Extensions
p.ParamProps = x.ParamProps
return nil
}
// MarshalJSON converts this items object to JSON
func (p Parameter) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(p.CommonValidations)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// PathItemProps the path item specific properties
@ -46,6 +48,10 @@ type PathItem struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *PathItem) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
if err := json.Unmarshal(data, &p.Refable); err != nil {
return err
}
@ -55,6 +61,31 @@ func (p *PathItem) UnmarshalJSON(data []byte) error {
return json.Unmarshal(data, &p.PathItemProps)
}
func (p *PathItem) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
PathItemProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
p.Extensions = x.Extensions
p.PathItemProps = x.PathItemProps
if err := p.Refable.Ref.fromMap(p.Extensions); err != nil {
return err
}
p.Extensions.sanitize()
if len(p.Extensions) == 0 {
p.Extensions = nil
}
return nil
}
// MarshalJSON converts this items object to JSON
func (p PathItem) MarshalJSON() ([]byte, error) {
b3, err := json.Marshal(p.Refable)

View File

@ -16,9 +16,12 @@ package spec
import (
"encoding/json"
"fmt"
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Paths holds the relative paths to the individual endpoints.
@ -34,6 +37,10 @@ type Paths struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (p *Paths) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, p)
}
var res map[string]json.RawMessage
if err := json.Unmarshal(data, &res); err != nil {
return err
@ -63,6 +70,58 @@ func (p *Paths) UnmarshalJSON(data []byte) error {
return nil
}
func (p *Paths) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
tok, err := dec.ReadToken()
if err != nil {
return err
}
var ext any
var pi PathItem
switch k := tok.Kind(); k {
case 'n':
return nil // noop
case '{':
for {
tok, err := dec.ReadToken()
if err != nil {
return err
}
if tok.Kind() == '}' {
return nil
}
switch k := tok.String(); {
case isExtensionKey(k):
ext = nil
if err := opts.UnmarshalNext(dec, &ext); err != nil {
return err
}
if p.Extensions == nil {
p.Extensions = make(map[string]any)
}
p.Extensions[k] = ext
case len(k) > 0 && k[0] == '/':
pi = PathItem{}
if err := opts.UnmarshalNext(dec, &pi); err != nil {
return err
}
if p.Paths == nil {
p.Paths = make(map[string]PathItem)
}
p.Paths[k] = pi
default:
_, err := dec.ReadValue() // skip value
return err
}
}
default:
return fmt.Errorf("unknown JSON kind: %v", k)
}
}
// MarshalJSON converts this items object to JSON
func (p Paths) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(p.VendorExtensible)

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// ResponseProps properties specific to a response
@ -39,13 +41,46 @@ type Response struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (r *Response) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
if err := json.Unmarshal(data, &r.ResponseProps); err != nil {
return err
}
if err := json.Unmarshal(data, &r.Refable); err != nil {
return err
}
return json.Unmarshal(data, &r.VendorExtensible)
if err := json.Unmarshal(data, &r.VendorExtensible); err != nil {
return err
}
return nil
}
func (r *Response) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
ResponseProps
Extensions
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
r.Extensions = x.Extensions
r.ResponseProps = x.ResponseProps
if err := r.Refable.Ref.fromMap(r.Extensions); err != nil {
return err
}
r.Extensions.sanitize()
if len(r.Extensions) == 0 {
r.Extensions = nil
}
return nil
}
// MarshalJSON converts this items object to JSON

View File

@ -16,10 +16,13 @@ package spec
import (
"encoding/json"
"fmt"
"reflect"
"strconv"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Responses is a container for the expected responses of an operation.
@ -42,6 +45,10 @@ type Responses struct {
// UnmarshalJSON hydrates this items instance with the data from JSON
func (r *Responses) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
if err := json.Unmarshal(data, &r.ResponsesProps); err != nil {
return err
}
@ -90,21 +97,90 @@ func (r ResponsesProps) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals responses from JSON
func (r *ResponsesProps) UnmarshalJSON(data []byte) error {
var res map[string]Response
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, r)
}
var res map[string]json.RawMessage
if err := json.Unmarshal(data, &res); err != nil {
return nil
return err
}
if v, ok := res["default"]; ok {
r.Default = &v
value := Response{}
if err := json.Unmarshal(v, &value); err != nil {
return err
}
r.Default = &value
delete(res, "default")
}
for k, v := range res {
// Take all integral keys
if nk, err := strconv.Atoi(k); err == nil {
if r.StatusCodeResponses == nil {
r.StatusCodeResponses = map[int]Response{}
}
r.StatusCodeResponses[nk] = v
value := Response{}
if err := json.Unmarshal(v, &value); err != nil {
return err
}
r.StatusCodeResponses[nk] = value
}
}
return nil
}
func (r *Responses) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) (err error) {
tok, err := dec.ReadToken()
if err != nil {
return err
}
var ext any
var resp Response
switch k := tok.Kind(); k {
case 'n':
return nil // noop
case '{':
for {
tok, err := dec.ReadToken()
if err != nil {
return err
}
if tok.Kind() == '}' {
return nil
}
switch k := tok.String(); {
case isExtensionKey(k):
ext = nil
if err := opts.UnmarshalNext(dec, &ext); err != nil {
return err
}
if r.Extensions == nil {
r.Extensions = make(map[string]any)
}
r.Extensions[k] = ext
case k == "default":
resp = Response{}
if err := opts.UnmarshalNext(dec, &resp); err != nil {
return err
}
respCopy := resp
r.ResponsesProps.Default = &respCopy
default:
if nk, err := strconv.Atoi(k); err == nil {
resp = Response{}
if err := opts.UnmarshalNext(dec, &resp); err != nil {
return err
}
if r.StatusCodeResponses == nil {
r.StatusCodeResponses = map[int]Response{}
}
r.StatusCodeResponses[nk] = resp
}
}
}
default:
return fmt.Errorf("unknown JSON kind: %v", k)
}
}

View File

@ -21,6 +21,8 @@ import (
"strings"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// BooleanProperty creates a boolean property
@ -465,6 +467,10 @@ func (s Schema) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (s *Schema) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
props := struct {
SchemaProps
SwaggerSchemaProps
@ -511,3 +517,38 @@ func (s *Schema) UnmarshalJSON(data []byte) error {
return nil
}
func (s *Schema) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
SchemaProps
SwaggerSchemaProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
if err := x.Ref.fromMap(x.Extensions); err != nil {
return err
}
if err := x.Schema.fromMap(x.Extensions); err != nil {
return err
}
delete(x.Extensions, "$ref")
delete(x.Extensions, "$schema")
for _, pn := range swag.DefaultJSONNameProvider.GetJSONNames(s) {
delete(x.Extensions, pn)
}
if len(x.Extensions) == 0 {
x.Extensions = nil
}
s.ExtraProps = x.Extensions.sanitizeWithExtra()
s.VendorExtensible.Extensions = x.Extensions
s.SchemaProps = x.SchemaProps
s.SwaggerSchemaProps = x.SwaggerSchemaProps
return nil
}

View File

@ -18,6 +18,7 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// SecuritySchemeProps describes a swagger security scheme in the securityDefinitions section
@ -62,3 +63,20 @@ func (s *SecurityScheme) UnmarshalJSON(data []byte) error {
}
return json.Unmarshal(data, &s.VendorExtensible)
}
func (s *SecurityScheme) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
SecuritySchemeProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
s.VendorExtensible.Extensions = x.Extensions
s.SecuritySchemeProps = x.SecuritySchemeProps
return nil
}

View File

@ -19,6 +19,8 @@ import (
"fmt"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// Swagger this is the root document object for the API specification.
@ -46,6 +48,10 @@ func (s Swagger) MarshalJSON() ([]byte, error) {
// UnmarshalJSON unmarshals a swagger spec from json
func (s *Swagger) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var sw Swagger
if err := json.Unmarshal(data, &sw.SwaggerProps); err != nil {
return err
@ -57,6 +63,30 @@ func (s *Swagger) UnmarshalJSON(data []byte) error {
return nil
}
func (s *Swagger) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
// Note: If you're willing to make breaking changes, it is possible to
// optimize this and other usages of this pattern:
// https://github.com/kubernetes/kube-openapi/pull/319#discussion_r983165948
var x struct {
Extensions
SwaggerProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
s.Extensions = x.Extensions
s.SwaggerProps = x.SwaggerProps
s.Extensions.sanitize()
if len(s.Extensions) == 0 {
s.Extensions = nil
}
return nil
}
// SwaggerProps captures the top-level properties of an Api specification
//
// NOTE: validation rules
@ -108,6 +138,10 @@ func (s SchemaOrBool) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this bool or schema object from a JSON structure
func (s *SchemaOrBool) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var nw SchemaOrBool
if len(data) >= 4 {
if data[0] == '{' {
@ -123,6 +157,26 @@ func (s *SchemaOrBool) UnmarshalJSON(data []byte) error {
return nil
}
func (s *SchemaOrBool) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch k := dec.PeekKind(); k {
case '{':
err := opts.UnmarshalNext(dec, &s.Schema)
if err != nil {
return err
}
s.Allows = true
return nil
case 't', 'f':
err := opts.UnmarshalNext(dec, &s.Allows)
if err != nil {
return err
}
return nil
default:
return fmt.Errorf("expected object or bool, not '%v'", k.String())
}
}
// SchemaOrStringArray represents a schema or a string array
type SchemaOrStringArray struct {
Schema *Schema
@ -142,6 +196,10 @@ func (s SchemaOrStringArray) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this schema object or array from a JSON structure
func (s *SchemaOrStringArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var first byte
if len(data) > 1 {
first = data[0]
@ -163,6 +221,18 @@ func (s *SchemaOrStringArray) UnmarshalJSON(data []byte) error {
return nil
}
func (s *SchemaOrStringArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch dec.PeekKind() {
case '{':
return opts.UnmarshalNext(dec, &s.Schema)
case '[':
return opts.UnmarshalNext(dec, &s.Property)
default:
_, err := dec.ReadValue()
return err
}
}
// Definitions contains the models explicitly defined in this spec
// An object to hold data types that can be consumed and produced by operations.
// These data types can be primitives, arrays or models.
@ -193,6 +263,10 @@ func (s StringOrArray) Contains(value string) bool {
// UnmarshalJSON unmarshals this string or array object from a JSON array or JSON string
func (s *StringOrArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var first byte
if len(data) > 1 {
first = data[0]
@ -223,6 +297,23 @@ func (s *StringOrArray) UnmarshalJSON(data []byte) error {
}
}
func (s *StringOrArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch k := dec.PeekKind(); k {
case '[':
*s = StringOrArray{}
return opts.UnmarshalNext(dec, (*[]string)(s))
case '"':
*s = StringOrArray{""}
return opts.UnmarshalNext(dec, &(*s)[0])
case 'n':
// Throw out null token
_, _ = dec.ReadToken()
return nil
default:
return fmt.Errorf("expected string or array, not '%v'", k.String())
}
}
// MarshalJSON converts this string or array to a JSON array or JSON string
func (s StringOrArray) MarshalJSON() ([]byte, error) {
if len(s) == 1 {
@ -264,6 +355,10 @@ func (s SchemaOrArray) MarshalJSON() ([]byte, error) {
// UnmarshalJSON converts this schema object or array from a JSON structure
func (s *SchemaOrArray) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, s)
}
var nw SchemaOrArray
var first byte
if len(data) > 1 {
@ -284,3 +379,15 @@ func (s *SchemaOrArray) UnmarshalJSON(data []byte) error {
*s = nw
return nil
}
func (s *SchemaOrArray) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
switch dec.PeekKind() {
case '{':
return opts.UnmarshalNext(dec, &s.Schema)
case '[':
return opts.UnmarshalNext(dec, &s.Schemas)
default:
_, err := dec.ReadValue()
return err
}
}

View File

@ -18,6 +18,8 @@ import (
"encoding/json"
"github.com/go-openapi/swag"
"k8s.io/kube-openapi/pkg/internal"
jsonv2 "k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json"
)
// TagProps describe a tag entry in the top level tags section of a swagger spec
@ -52,8 +54,29 @@ func (t Tag) MarshalJSON() ([]byte, error) {
// UnmarshalJSON marshal this from JSON
func (t *Tag) UnmarshalJSON(data []byte) error {
if internal.UseOptimizedJSONUnmarshaling {
return jsonv2.Unmarshal(data, t)
}
if err := json.Unmarshal(data, &t.TagProps); err != nil {
return err
}
return json.Unmarshal(data, &t.VendorExtensible)
}
func (t *Tag) UnmarshalNextJSON(opts jsonv2.UnmarshalOptions, dec *jsonv2.Decoder) error {
var x struct {
Extensions
TagProps
}
if err := opts.UnmarshalNext(dec, &x); err != nil {
return err
}
x.Extensions.sanitize()
if len(x.Extensions) == 0 {
x.Extensions = nil
}
t.VendorExtensible.Extensions = x.Extensions
t.TagProps = x.TagProps
return nil
}

12
vendor/modules.txt vendored
View File

@ -378,8 +378,8 @@ github.com/kubernetes-csi/csi-lib-utils/connection
github.com/kubernetes-csi/csi-lib-utils/metrics
github.com/kubernetes-csi/csi-lib-utils/protosanitizer
github.com/kubernetes-csi/csi-lib-utils/rpc
# github.com/kubernetes-csi/external-snapshotter/client/v6 v6.1.0
## explicit; go 1.18
# github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0
## explicit; go 1.19
github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1
github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/scheme
github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1
@ -800,7 +800,7 @@ gopkg.in/yaml.v2
# gopkg.in/yaml.v3 v3.0.1
## explicit
gopkg.in/yaml.v3
# k8s.io/api v0.25.4 => k8s.io/api v0.25.4
# k8s.io/api v0.26.0 => k8s.io/api v0.25.4
## explicit; go 1.19
k8s.io/api/admission/v1
k8s.io/api/admission/v1beta1
@ -854,7 +854,7 @@ k8s.io/api/storage/v1beta1
## explicit; go 1.19
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions
k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
# k8s.io/apimachinery v0.25.4 => k8s.io/apimachinery v0.25.4
# k8s.io/apimachinery v0.26.0 => k8s.io/apimachinery v0.25.4
## explicit; go 1.19
k8s.io/apimachinery/pkg/api/equality
k8s.io/apimachinery/pkg/api/errors
@ -1229,12 +1229,14 @@ k8s.io/klog/v2/internal/clock
k8s.io/klog/v2/internal/dbg
k8s.io/klog/v2/internal/serialize
k8s.io/klog/v2/internal/severity
# k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea
# k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280
## explicit; go 1.18
k8s.io/kube-openapi/pkg/builder3/util
k8s.io/kube-openapi/pkg/common
k8s.io/kube-openapi/pkg/handler3
k8s.io/kube-openapi/pkg/internal
k8s.io/kube-openapi/pkg/internal/handler
k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json
k8s.io/kube-openapi/pkg/openapiconv
k8s.io/kube-openapi/pkg/schemaconv
k8s.io/kube-openapi/pkg/schemamutation