diff --git a/README.md b/README.md index 66f23ee0d..0384f64f6 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/ceph/ceph-csi)](https://goreportcard.com/report/github.com/ceph/ceph-csi) [![TODOs](https://badgen.net/https/api.tickgit.com/badgen/github.com/ceph/ceph-csi/devel)](https://www.tickgit.com/browse?repo=github.com/ceph/ceph-csi&branch=devel) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5940/badge)](https://bestpractices.coreinfrastructure.org/projects/5940) - [Ceph CSI](#ceph-csi) - [Overview](#overview) @@ -54,6 +55,7 @@ environments. | Ceph CSI Version | Container Orchestrator Name | Version Tested| | -----------------| --------------------------- | --------------| +| v3.6.1 | Kubernetes | v1.21, v1.22, v1.23| | v3.6.0 | Kubernetes | v1.21, v1.22, v1.23| | v3.5.1 | Kubernetes | v1.21, v1.22, v1.23| | v3.5.0 | Kubernetes | v1.21, v1.22, v1.23| @@ -125,6 +127,7 @@ in the Kubernetes documentation. | Ceph CSI Release/Branch | Container image name | Image Tag | | ----------------------- | ---------------------------- | --------- | | devel (Branch) | quay.io/cephcsi/cephcsi | canary | +| v3.6.1 (Release) | quay.io/cephcsi/cephcsi | v3.6.1 | | v3.6.0 (Release) | quay.io/cephcsi/cephcsi | v3.6.0 | | v3.5.1 (Release) | quay.io/cephcsi/cephcsi | v3.5.1 | | v3.5.0 (Release) | quay.io/cephcsi/cephcsi | v3.5.0 | @@ -175,14 +178,14 @@ A regular dev standup takes place every [Monday,Tuesday and Thursday at timezone by executing command `date -d "12:00 UTC"` on terminal Any changes to the meeting schedule will be added to the [agenda -doc](https://docs.google.com/document/d/1K1aerdMpraIh56-skdoEoVF9RZrO4NUcbHtjN-f3u1s). +doc](https://hackmd.io/6GL90WFGQL-L4DcIfIAKeQ). Anyone who wants to discuss the direction of the project, design and implementation reviews, or general questions with the broader community is welcome and encouraged to join. - Meeting link: -- [Current agenda](https://docs.google.com/document/d/1K1aerdMpraIh56-skdoEoVF9RZrO4NUcbHtjN-f3u1s) +- [Current agenda](https://hackmd.io/6GL90WFGQL-L4DcIfIAKeQ) ## Contact diff --git a/actions/retest/main.go b/actions/retest/main.go index a3b85042e..cc02cd4c2 100644 --- a/actions/retest/main.go +++ b/actions/retest/main.go @@ -181,8 +181,8 @@ func main() { } if failedTestFound { - // comment `@Mergifyio refresh` so mergifyio adds the pr back into the queue. - msg := "@Mergifyio refresh" + // comment `@Mergifyio requeue` so mergifyio adds the pr back into the queue. + msg := "@Mergifyio requeue" comment := &github.IssueComment{ Body: github.String(msg), } diff --git a/build.env b/build.env index fb26cad81..ee029cb3b 100644 --- a/build.env +++ b/build.env @@ -35,7 +35,7 @@ SNAPSHOT_VERSION=v5.0.1 #GO_COVER_DIR=_output/ # helm chart generation, testing and publishing -HELM_VERSION=v3.1.2 +HELM_VERSION=v3.8.2 # minikube settings MINIKUBE_VERSION=v1.25.2 diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index 181a019f1..45cc1d3f6 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -90,7 +90,7 @@ charts and their default values. | `nodeplugin.updateStrategy` | Specifies the update Strategy. If you are using ceph-fuse client set this value to OnDelete | `RollingUpdate` | | `nodeplugin.priorityClassName` | Set user created priorityclassName for csi plugin pods. default is system-node-critical which is highest priority | `system-node-critical` | | `nodeplugin.profiling.enabled` | Specifies whether profiling should be enabled | `false` | -| `nodeplugin.registrar.image.repository` | Node-Registrar image repository URL | `k8s.gcr.io/sig-storage/csi-node-driver-registrar` | +| `nodeplugin.registrar.image.repository` | Node-Registrar image repository URL | `registry.k8s.io/sig-storage/csi-node-driver-registrar` | | `nodeplugin.registrar.image.tag` | Image tag | `v2.2.0` | | `nodeplugin.registrar.image.pullPolicy` | Image pull policy | `IfNotPresent` | | `nodeplugin.plugin.image.repository` | Nodeplugin image repository URL | `quay.io/cephcsi/cephcsi` | @@ -105,20 +105,20 @@ charts and their default values. | `provisioner.timeout` | GRPC timeout for waiting for creation or deletion of a volume | `60s` | | `provisioner.priorityClassName` | Set user created priorityclassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical` | | `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | -| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `k8s.gcr.io/sig-storage/csi-provisioner` | +| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | | `provisioner.provisioner.image.tag` | Specifies image tag | `v2.2.2` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `k8s.gcr.io/sig-storage/csi-attacher` | +| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `registry.k8s.io/sig-storage/csi-attacher` | | `provisioner.attacher.image.tag` | Specifies image tag | `v3.2.1` | | `provisioner.attacher.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | | `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | -| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `k8s.gcr.io/sig-storage/csi-resizer` | +| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `registry.k8s.io/sig-storage/csi-resizer` | | `provisioner.resizer.image.tag` | Specifies image tag | `v1.2.0` | | `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | | `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | -| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `k8s.gcr.io/sig-storage/csi-snapshotter` | +| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | | `provisioner.snapshotter.image.tag` | Specifies image tag | `v4.1.1` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml deleted file mode 100644 index f5bb71ef9..000000000 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrole.yaml +++ /dev/null @@ -1,18 +0,0 @@ -{{- if .Values.rbac.create -}} -{{- if .Values.topology.enabled }} -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] -{{- end }} -{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml deleted file mode 100644 index f394bab87..000000000 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-clusterrolebinding.yaml +++ /dev/null @@ -1,22 +0,0 @@ -{{- if .Values.rbac.create -}} -{{- if .Values.topology.enabled }} -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} - labels: - app: {{ include "ceph-csi-cephfs.name" . }} - chart: {{ include "ceph-csi-cephfs.chart" . }} - component: {{ .Values.nodeplugin.name }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subjects: - - kind: ServiceAccount - name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} - namespace: {{ .Release.Namespace }} -roleRef: - kind: ClusterRole - name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} - apiGroup: rbac.authorization.k8s.io -{{- end }} -{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 58e61e93d..0b997a16b 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -42,6 +42,7 @@ spec: # created by privileged CSI driver container. securityContext: privileged: true + allowPrivilegeEscalation: true image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} args: @@ -135,6 +136,7 @@ spec: - name: liveness-prometheus securityContext: privileged: true + allowPrivilegeEscalation: true image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} args: diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 73f396f30..b23246a13 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -27,7 +27,7 @@ serviceAccounts: # - "" # cephFS: # subvolumeGroup: "csi" -# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" +# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" csiConfig: [] # Set logging level for csi containers. @@ -80,7 +80,7 @@ nodeplugin: registrar: image: - repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar tag: v2.4.0 pullPolicy: IfNotPresent resources: {} @@ -161,7 +161,7 @@ provisioner: provisioner: image: - repository: k8s.gcr.io/sig-storage/csi-provisioner + repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.1.0 pullPolicy: IfNotPresent resources: {} @@ -170,7 +170,7 @@ provisioner: name: attacher enabled: true image: - repository: k8s.gcr.io/sig-storage/csi-attacher + repository: registry.k8s.io/sig-storage/csi-attacher tag: v3.4.0 pullPolicy: IfNotPresent resources: {} @@ -179,14 +179,14 @@ provisioner: name: resizer enabled: true image: - repository: k8s.gcr.io/sig-storage/csi-resizer + repository: registry.k8s.io/sig-storage/csi-resizer tag: v1.4.0 pullPolicy: IfNotPresent resources: {} snapshotter: image: - repository: k8s.gcr.io/sig-storage/csi-snapshotter + repository: registry.k8s.io/sig-storage/csi-snapshotter tag: v4.2.0 pullPolicy: IfNotPresent resources: {} diff --git a/charts/ceph-csi-rbd/README.md b/charts/ceph-csi-rbd/README.md index a0e79d263..4c4542bee 100644 --- a/charts/ceph-csi-rbd/README.md +++ b/charts/ceph-csi-rbd/README.md @@ -92,7 +92,7 @@ charts and their default values. | `nodeplugin.updateStrategy` | Specifies the update Strategy. If you are using ceph-fuse client set this value to OnDelete | `RollingUpdate` | | `nodeplugin.priorityClassName` | Set user created priorityclassName for csi plugin pods. default is system-node-critical which is highest priority | `system-node-critical` | | `nodeplugin.profiling.enabled` | Specifies whether profiling should be enabled | `false` | -| `nodeplugin.registrar.image.repository` | Node Registrar image repository URL | `k8s.gcr.io/sig-storage/csi-node-driver-registrar` | +| `nodeplugin.registrar.image.repository` | Node Registrar image repository URL | `registry.k8s.io/sig-storage/csi-node-driver-registrar` | | `nodeplugin.registrar.image.tag` | Image tag | `v2.2.0` | | `nodeplugin.registrar.image.pullPolicy` | Image pull policy | `IfNotPresent` | | `nodeplugin.plugin.image.repository` | Nodeplugin image repository URL | `quay.io/cephcsi/cephcsi` | @@ -113,20 +113,20 @@ charts and their default values. | `provisioner.timeout` | GRPC timeout for waiting for creation or deletion of a volume | `60s` | | `provisioner.priorityClassName` | Set user created priorityclassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical` | | `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | -| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `k8s.gcr.io/sig-storage/csi-provisioner` | +| `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | | `provisioner.provisioner.image.tag` | Specifies image tag | `v2.2.2` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `k8s.gcr.io/sig-storage/csi-attacher` | +| `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `registry.k8s.io/sig-storage/csi-attacher` | | `provisioner.attacher.image.tag` | Specifies image tag | `v3.2.1` | | `provisioner.attacher.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | | `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | -| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `k8s.gcr.io/sig-storage/csi-resizer` | +| `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `registry.k8s.io/sig-storage/csi-resizer` | | `provisioner.resizer.image.tag` | Specifies image tag | `v1.2.0` | | `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | | `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | -| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `k8s.gcr.io/sig-storage/csi-snapshotter` | +| `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | | `provisioner.snapshotter.image.tag` | Specifies image tag | `v4.1.1` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml index 120d9627c..decd6e78c 100644 --- a/charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -42,6 +42,7 @@ spec: # created by privileged CSI driver container. securityContext: privileged: true + allowPrivilegeEscalation: true image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" imagePullPolicy: {{ .Values.nodeplugin.registrar.image.pullPolicy }} args: @@ -142,6 +143,7 @@ spec: - name: liveness-prometheus securityContext: privileged: true + allowPrivilegeEscalation: true image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} args: diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index b6e6e55e4..82a5462dc 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -25,7 +25,8 @@ serviceAccounts: # monitors: # - "" # - "" -# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" +# rbd: +# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" csiConfig: [] # Configuration details of clusterID,PoolID and FscID mapping @@ -102,7 +103,7 @@ nodeplugin: registrar: image: - repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar tag: v2.4.0 pullPolicy: IfNotPresent resources: {} @@ -198,7 +199,7 @@ provisioner: provisioner: image: - repository: k8s.gcr.io/sig-storage/csi-provisioner + repository: registry.k8s.io/sig-storage/csi-provisioner tag: v3.1.0 pullPolicy: IfNotPresent resources: {} @@ -207,7 +208,7 @@ provisioner: name: attacher enabled: true image: - repository: k8s.gcr.io/sig-storage/csi-attacher + repository: registry.k8s.io/sig-storage/csi-attacher tag: v3.4.0 pullPolicy: IfNotPresent resources: {} @@ -216,14 +217,14 @@ provisioner: name: resizer enabled: true image: - repository: k8s.gcr.io/sig-storage/csi-resizer + repository: registry.k8s.io/sig-storage/csi-resizer tag: v1.4.0 pullPolicy: IfNotPresent resources: {} snapshotter: image: - repository: k8s.gcr.io/sig-storage/csi-snapshotter + repository: registry.k8s.io/sig-storage/csi-snapshotter tag: v4.2.0 pullPolicy: IfNotPresent resources: {} diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index a690a230d..c54b017e8 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -43,7 +43,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=5" @@ -60,7 +60,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 args: - "--csi-address=$(ADDRESS)" - "--v=5" @@ -76,7 +76,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1 args: - "--csi-address=$(ADDRESS)" - "--v=5" @@ -90,7 +90,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-cephfsplugin-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml index e8e7f1cf2..5c35248f6 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml @@ -26,7 +26,8 @@ spec: # created by privileged CSI driver container. securityContext: privileged: true - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0 + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" @@ -106,6 +107,7 @@ spec: - name: liveness-prometheus securityContext: privileged: true + allowPrivilegeEscalation: true image: quay.io/cephcsi/cephcsi:canary args: - "--type=liveness" diff --git a/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml index 1c1ccda3f..a1ee7d1a0 100644 --- a/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml @@ -3,25 +3,3 @@ apiVersion: v1 kind: ServiceAccount metadata: name: cephfs-csi-nodeplugin ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: cephfs-csi-nodeplugin -subjects: - - kind: ServiceAccount - name: cephfs-csi-nodeplugin - namespace: default -roleRef: - kind: ClusterRole - name: cephfs-csi-nodeplugin - apiGroup: rbac.authorization.k8s.io diff --git a/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml b/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml index d2155690c..4147b33af 100644 --- a/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml +++ b/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml @@ -40,7 +40,7 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=5" diff --git a/deploy/nfs/kubernetes/csi-nfsplugin.yaml b/deploy/nfs/kubernetes/csi-nfsplugin.yaml index ae0989a2a..1f3007283 100644 --- a/deploy/nfs/kubernetes/csi-nfsplugin.yaml +++ b/deploy/nfs/kubernetes/csi-nfsplugin.yaml @@ -18,7 +18,7 @@ spec: - --probe-timeout=3s - --health-port=29653 - --v=2 - image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.5.0 imagePullPolicy: IfNotPresent name: liveness-probe resources: @@ -44,7 +44,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: spec.nodeName - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0 imagePullPolicy: IfNotPresent livenessProbe: exec: @@ -86,7 +86,7 @@ spec: fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock - image: mcr.microsoft.com/k8s/csi/nfs-csi:v3.1.0 + image: registry.k8s.io/sig-storage/nfsplugin:v3.1.0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 5 diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index ce364636a..7cc0b7317 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -47,7 +47,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.1.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=5" @@ -67,7 +67,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1 args: - "--csi-address=$(ADDRESS)" - "--v=5" @@ -82,7 +82,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-attacher - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + image: registry.k8s.io/sig-storage/csi-attacher:v3.4.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" @@ -96,7 +96,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-resizer - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0 args: - "--csi-address=$(ADDRESS)" - "--v=5" diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index e9117e9ff..3b21ae428 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -28,7 +28,8 @@ spec: # created by privileged CSI driver container. securityContext: privileged: true - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.4.0 + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0 args: - "--v=5" - "--csi-address=/csi/csi.sock" @@ -124,6 +125,7 @@ spec: - name: liveness-prometheus securityContext: privileged: true + allowPrivilegeEscalation: true image: quay.io/cephcsi/cephcsi:canary args: - "--type=liveness" diff --git a/docs/ceph-csi-upgrade.md b/docs/ceph-csi-upgrade.md index b12fd3a4a..2a641794d 100644 --- a/docs/ceph-csi-upgrade.md +++ b/docs/ceph-csi-upgrade.md @@ -90,12 +90,12 @@ compatibility support and without prior notice. **Also, we do not recommend any direct upgrades to 3.6 except from 3.5 to 3.6.** For example, upgrading from 3.4 to 3.6 is not recommended. -git checkout v3.6.0 tag +git checkout v3.6.1 tag ```bash git clone https://github.com/ceph/ceph-csi.git cd ./ceph-csi -git checkout v3.6.0 +git checkout v3.6.1 ``` ```console @@ -155,9 +155,6 @@ nodeplugin daemonset ```bash $ kubectl apply -f deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml serviceaccount/cephfs-csi-nodeplugin configured -clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin configured -clusterrole.rbac.authorization.k8s.io/cephfs-csi-nodeplugin-rules configured -clusterrolebinding.rbac.authorization.k8s.io/cephfs-csi-nodeplugin configured ``` If you determined in [Pre-upgrade considerations](#pre-upgrade-considerations) diff --git a/docs/design/proposals/clusterid-mapping.md b/docs/design/proposals/clusterid-mapping.md index 4f45e05d5..182b87e3a 100644 --- a/docs/design/proposals/clusterid-mapping.md +++ b/docs/design/proposals/clusterid-mapping.md @@ -44,7 +44,9 @@ data: [ { "clusterID": "rook-ceph", - "radosNamespace": "", + "rbd": { + "radosNamespace": "", + }, "monitors": [ "192.168.39.82:6789" ], @@ -54,7 +56,9 @@ data: }, { "clusterID": "fs-id", - "radosNamespace": "", + "rbd": { + "radosNamespace": "", + }, "monitors": [ "192.168.39.83:6789" ], diff --git a/e2e/README.md b/e2e/README.md index d10783292..3436aca6b 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -54,7 +54,7 @@ following environment variables can be exported to customize kubernetes deployme | MEMORY | Amount of RAM allocated to the minikube VM in MB | 4096 | | VM_DRIVER | VM driver to create virtual machine | virtualbox | | CEPHCSI_IMAGE_REPO | Repo URL to pull cephcsi images | quay.io/cephcsi | -| K8S_IMAGE_REPO | Repo URL to pull kubernetes sidecar images | k8s.gcr.io/sig-storage | +| K8S_IMAGE_REPO | Repo URL to pull kubernetes sidecar images | registry.k8s.io/sig-storage | | K8S_FEATURE_GATES | Feature gates to enable on kubernetes cluster | BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true | | ROOK_BLOCK_POOL_NAME | Block pool name to create in the rook instance | newrbdpool | @@ -87,22 +87,23 @@ Thanks to [minikube](../scripts/minikube.sh) script for the handy `deploy-rook` In addition to standard go tests parameters, the following custom parameters are available while running tests: -| flag | description | -| ----------------- | ----------------------------------------------------------------------------- | -| deploy-timeout | Timeout to wait for created kubernetes resources (default: 10 minutes) | -| deploy-cephfs | Deploy cephFS CSI driver as part of E2E (default: true) | -| deploy-rbd | Deploy rbd CSI driver as part of E2E (default: true) | -| test-cephfs | Test cephFS CSI driver as part of E2E (default: true) | -| upgrade-testing | Perform upgrade testing (default: false) | -| upgrade-version | Target version for upgrade testing (default: "v3.5.1") | -| test-rbd | Test rbd CSI driver as part of E2E (default: true) | -| cephcsi-namespace | The namespace in which cephcsi driver will be created (default: "default") | -| rook-namespace | The namespace in which rook operator is installed (default: "rook-ceph") | -| kubeconfig | Path to kubeconfig containing embedded authinfo (default: $HOME/.kube/config) | -| timeout | Panic test binary after duration d (default 0, timeout disabled) | -| v | Verbose: print additional output | -| filesystem | Name of the CephFS filesystem (default: "myfs") | -| is-openshift | Run in OpenShift compatibility mode, skips certain new feature tests | +| flag | description | +| ----------------- | ------------------------------------------------------------------------------------------------- | +| deploy-timeout | Timeout to wait for created kubernetes resources (default: 10 minutes) | +| deploy-cephfs | Deploy cephFS CSI driver as part of E2E (default: true) | +| deploy-rbd | Deploy rbd CSI driver as part of E2E (default: true) | +| test-cephfs | Test cephFS CSI driver as part of E2E (default: true) | +| upgrade-testing | Perform upgrade testing (default: false) | +| upgrade-version | Target version for upgrade testing (default: "v3.5.1") | +| test-rbd | Test rbd CSI driver as part of E2E (default: true) | +| cephcsi-namespace | The namespace in which cephcsi driver will be created (default: "default") | +| rook-namespace | The namespace in which rook operator is installed (default: "rook-ceph") | +| kubeconfig | Path to kubeconfig containing embedded authinfo (default: $HOME/.kube/config) | +| timeout | Panic test binary after duration d (default 0, timeout disabled) | +| v | Verbose: print additional output | +| is-openshift | Run in OpenShift compatibility mode, skips certain new feature tests | +| filesystem | Name of the CephFS filesystem (default: "myfs") | +| clusterid | Use the Ceph cluster id in the StorageClasses and SnapshotClasses (default: `ceph fsid` detected) | ## E2E for snapshot diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 4b42b75a1..1c9671243 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -19,7 +19,6 @@ package e2e import ( "context" "fmt" - "os" "strings" "sync" @@ -69,86 +68,48 @@ func deleteCephfsPlugin() { } func createORDeleteCephfsResources(action kubectlAction) { - csiDriver, err := os.ReadFile(cephFSDirPath + csiDriverObject) - if err != nil { - // createORDeleteRbdResources is used for upgrade testing as csidriverObject is - // newly added, discarding file not found error. - if !os.IsNotExist(err) { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+csiDriverObject, err) - } - } else { - err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout) + resources := []ResourceDeployer{ + &yamlResource{ + filename: cephFSDirPath + csiDriverObject, + allowMissing: true, + }, + &yamlResource{ + filename: examplePath + cephConfconfigMap, + allowMissing: true, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSProvisioner, + namespace: cephCSINamespace, + oneReplica: true, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSProvisionerRBAC, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSProvisionerPSP, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSNodePlugin, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSNodePluginRBAC, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: cephFSDirPath + cephFSNodePluginPSP, + namespace: cephCSINamespace, + }, + } + + for _, r := range resources { + err := r.Do(action) if err != nil { - e2elog.Failf("failed to %s CSIDriver object: %v", action, err) + e2elog.Failf("failed to %s resource: %v", action, err) } } - cephConf, err := os.ReadFile(examplePath + cephConfconfigMap) - if err != nil { - // createORDeleteCephfsResources is used for upgrade testing as cephConfConfigmap is - // newly added, discarding file not found error. - if !os.IsNotExist(err) { - e2elog.Failf("failed to read content from %s: %v", examplePath+cephConfconfigMap, err) - } - } else { - err = retryKubectlInput(cephCSINamespace, action, string(cephConf), deployTimeout) - if err != nil { - e2elog.Failf("failed to %s ceph-conf configmap object: %v", action, err) - } - } - data, err := replaceNamespaceInTemplate(cephFSDirPath + cephFSProvisioner) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSProvisioner, err) - } - data = oneReplicaDeployYaml(data) - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS provisioner: %v", action, err) - } - data, err = replaceNamespaceInTemplate(cephFSDirPath + cephFSProvisionerRBAC) - - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSProvisionerRBAC, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS provisioner rbac: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(cephFSDirPath + cephFSProvisionerPSP) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSProvisionerPSP, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS provisioner psp: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(cephFSDirPath + cephFSNodePlugin) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSNodePlugin, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS nodeplugin: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(cephFSDirPath + cephFSNodePluginRBAC) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSNodePluginRBAC, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS nodeplugin rbac: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(cephFSDirPath + cephFSNodePluginPSP) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", cephFSDirPath+cephFSNodePluginPSP, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s CephFS nodeplugin psp: %v", action, err) - } } func validateSubvolumeCount(f *framework.Framework, count int, fileSystemName, subvolumegroup string) { @@ -277,6 +238,10 @@ var _ = Describe("cephfs", func() { }) Context("Test CephFS CSI", func() { + if !testCephFS || upgradeTesting { + return + } + It("Test CephFS CSI", func() { pvcPath := cephFSExamplePath + "pvc.yaml" appPath := cephFSExamplePath + "pod.yaml" @@ -296,7 +261,7 @@ var _ = Describe("cephfs", func() { } }) - By("checking nodeplugin deamonset pods are running", func() { + By("checking nodeplugin daemonset pods are running", func() { err := waitForDaemonSets(cephFSDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout) if err != nil { e2elog.Failf("timeout waiting for daemonset %s: %v", cephFSDeamonSetName, err) @@ -374,6 +339,11 @@ var _ = Describe("cephfs", func() { err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { + if rwopMayFail(err) { + e2elog.Logf("RWOP is not supported: %v", err) + + return + } e2elog.Failf("failed to create PVC: %v", err) } err = createApp(f.ClientSet, app, deployTimeout) diff --git a/e2e/cephfs_helper.go b/e2e/cephfs_helper.go index 7ceadf6d8..e8bc84c72 100644 --- a/e2e/cephfs_helper.go +++ b/e2e/cephfs_helper.go @@ -26,7 +26,6 @@ import ( snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" v1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -93,14 +92,12 @@ func createCephfsStorageClass( // fetch and set fsID from the cluster if not set in params if _, found := params["clusterID"]; !found { - fsID, stdErr, failErr := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) - if failErr != nil { - return failErr + var fsID string + fsID, err = getClusterID(f) + if err != nil { + return fmt.Errorf("failed to get clusterID: %w", err) } - if stdErr != "" { - return fmt.Errorf("error getting fsid %v", stdErr) - } - sc.Parameters["clusterID"] = strings.Trim(fsID, "\n") + sc.Parameters["clusterID"] = fsID } sc.Namespace = cephCSINamespace @@ -109,13 +106,10 @@ func createCephfsStorageClass( return wait.PollImmediate(poll, timeout, func() (bool, error) { _, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{}) if err != nil { - e2elog.Logf("error creating StorageClass %q in namespace %q: %v", sc.Name, sc.Namespace, err) + e2elog.Logf("error creating StorageClass %q: %v", sc.Name, err) if isRetryableAPIError(err) { return false, nil } - if apierrs.IsNotFound(err) { - return false, nil - } return false, fmt.Errorf("failed to create StorageClass %q: %w", sc.Name, err) } diff --git a/e2e/configmap.go b/e2e/configmap.go index 1f90edb85..676968a3a 100644 --- a/e2e/configmap.go +++ b/e2e/configmap.go @@ -20,7 +20,6 @@ import ( "context" "encoding/json" "fmt" - "strings" "github.com/ceph/ceph-csi/internal/util" @@ -45,24 +44,25 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra return err } - fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + fsID, err := getClusterID(f) if err != nil { - return fmt.Errorf("failed to exec command in toolbox: %w", err) + return fmt.Errorf("failed to get clusterID: %w", err) } - if stdErr != "" { - return fmt.Errorf("error getting fsid %v", stdErr) - } - // remove new line present in fsID - fsID = strings.Trim(fsID, "\n") + // get mon list mons, err := getMons(rookNamespace, c) if err != nil { return err } conmap := []util.ClusterInfo{{ - ClusterID: fsID, - Monitors: mons, - RadosNamespace: radosNamespace, + ClusterID: fsID, + Monitors: mons, + RBD: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + RadosNamespace string `json:"radosNamespace"` + }{ + RadosNamespace: radosNamespace, + }, }} if upgradeTesting { subvolumegroup = "csi" @@ -132,7 +132,7 @@ func createCustomConfigMap( case "radosNamespace": for c := range conmap { if conmap[c].ClusterID == cluster { - conmap[c].RadosNamespace = j + conmap[c].RBD.RadosNamespace = j } } } diff --git a/e2e/deployment.go b/e2e/deployment.go index 8816a9b0a..a2f809d23 100644 --- a/e2e/deployment.go +++ b/e2e/deployment.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "os" "time" appsv1 "k8s.io/api/apps/v1" @@ -178,3 +179,78 @@ func waitForDeploymentComplete(clientSet kubernetes.Interface, name, ns string, return nil } + +// ResourceDeployer provides a generic interface for deploying different +// resources. +type ResourceDeployer interface { + // Do is used to create/delete a resource with kubectl. + Do(action kubectlAction) error +} + +// yamlResource reads a YAML file and creates/deletes the resource with +// kubectl. +type yamlResource struct { + filename string + + // allowMissing prevents a failure in case the file is missing. + allowMissing bool +} + +func (yr *yamlResource) Do(action kubectlAction) error { + data, err := os.ReadFile(yr.filename) + if err != nil { + if os.IsNotExist(err) && yr.allowMissing { + return nil + } + + return fmt.Errorf("failed to read content from %q: %w", yr.filename, err) + } + + err = retryKubectlInput(cephCSINamespace, action, string(data), deployTimeout) + if err != nil { + return fmt.Errorf("failed to %s resource %q: %w", action, yr.filename, err) + } + + return nil +} + +// yamlResourceNamespaced takes a filename and calls +// replaceNamespaceInTemplate() on it. There are several options for adjusting +// templates, each has their own comment. +type yamlResourceNamespaced struct { + filename string + namespace string + + // set the number of replicas in a Deployment to 1. + oneReplica bool + + // enable topology support (for RBD) + enableTopology bool + domainLabel string +} + +func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error { + data, err := replaceNamespaceInTemplate(yrn.filename) + if err != nil { + return fmt.Errorf("failed to read content from %q: %w", yrn.filename, err) + } + + if yrn.oneReplica { + data = oneReplicaDeployYaml(data) + } + + if yrn.enableTopology { + data = enableTopologyInTemplate(data) + } + + if yrn.domainLabel != "" { + data = addTopologyDomainsToDSYaml(data, yrn.domainLabel) + } + + err = retryKubectlInput(yrn.namespace, action, data, deployTimeout) + if err != nil { + return fmt.Errorf("failed to %s resource %q in namespace %q: %w", action, yrn.filename, yrn.namespace, err) + } + + return nil +} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index aa5bd3d69..fec5cc5cd 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -43,8 +43,9 @@ func init() { flag.StringVar(&upgradeVersion, "upgrade-version", "v3.5.1", "target version for upgrade testing") flag.StringVar(&cephCSINamespace, "cephcsi-namespace", defaultNs, "namespace in which cephcsi deployed") flag.StringVar(&rookNamespace, "rook-namespace", "rook-ceph", "namespace in which rook is deployed") - flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use") flag.BoolVar(&isOpenShift, "is-openshift", false, "disables certain checks on OpenShift") + flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use") + flag.StringVar(&clusterID, "clusterid", "", "Ceph cluster ID to use (defaults to `ceph fsid` detection)") setDefaultKubeconfig() // Register framework flags, then handle flags diff --git a/e2e/pvc.go b/e2e/pvc.go index a46765d5d..2f7575d29 100644 --- a/e2e/pvc.go +++ b/e2e/pvc.go @@ -339,7 +339,7 @@ func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t i return false, nil } if stdOut == "" { - e2elog.Logf("no metrics received from kublet on IP %s", kubelet) + e2elog.Logf("no metrics received from kubelet on IP %s", kubelet) return false, nil } diff --git a/e2e/rbd.go b/e2e/rbd.go index f708ca5d7..eddb7b9b2 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "os" "strings" "time" @@ -131,90 +130,50 @@ func deleteRBDPlugin() { } func createORDeleteRbdResources(action kubectlAction) { - csiDriver, err := os.ReadFile(rbdDirPath + csiDriverObject) - if err != nil { - // createORDeleteRbdResources is used for upgrade testing as csidriverObject is - // newly added, discarding file not found error. - if !os.IsNotExist(err) { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+csiDriverObject, err) - } - } else { - err = retryKubectlInput(cephCSINamespace, action, string(csiDriver), deployTimeout) + resources := []ResourceDeployer{ + &yamlResource{ + filename: rbdDirPath + csiDriverObject, + allowMissing: true, + }, + &yamlResource{ + filename: examplePath + cephConfconfigMap, + allowMissing: true, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdProvisioner, + namespace: cephCSINamespace, + oneReplica: true, + enableTopology: true, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdProvisionerRBAC, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdProvisionerPSP, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdNodePlugin, + namespace: cephCSINamespace, + domainLabel: nodeRegionLabel + "," + nodeZoneLabel, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdNodePluginRBAC, + namespace: cephCSINamespace, + }, + &yamlResourceNamespaced{ + filename: rbdDirPath + rbdNodePluginPSP, + namespace: cephCSINamespace, + }, + } + + for _, r := range resources { + err := r.Do(action) if err != nil { - e2elog.Failf("failed to %s CSIDriver object: %v", action, err) + e2elog.Failf("failed to %s resource: %v", action, err) } } - cephConf, err := os.ReadFile(examplePath + cephConfconfigMap) - if err != nil { - // createORDeleteRbdResources is used for upgrade testing as cephConf Configmap is - // newly added, discarding file not found error. - if !os.IsNotExist(err) { - e2elog.Failf("failed to read content from %s: %v", examplePath+cephConfconfigMap, err) - } - } else { - err = retryKubectlInput(cephCSINamespace, action, string(cephConf), deployTimeout) - if err != nil { - e2elog.Failf("failed to %s ceph-conf configmap object: %v", action, err) - } - } - data, err := replaceNamespaceInTemplate(rbdDirPath + rbdProvisioner) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdProvisioner, err) - } - data = oneReplicaDeployYaml(data) - data = enableTopologyInTemplate(data) - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s rbd provisioner: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerRBAC) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdProvisionerRBAC, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s provisioner rbac: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerPSP) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdProvisionerPSP, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s provisioner psp: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePlugin) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdNodePlugin, err) - } - - domainLabel := nodeRegionLabel + "," + nodeZoneLabel - data = addTopologyDomainsToDSYaml(data, domainLabel) - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s nodeplugin: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginRBAC) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdNodePluginRBAC, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s nodeplugin rbac: %v", action, err) - } - - data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginPSP) - if err != nil { - e2elog.Failf("failed to read content from %s: %v", rbdDirPath+rbdNodePluginPSP, err) - } - err = retryKubectlInput(cephCSINamespace, action, data, deployTimeout) - if err != nil { - e2elog.Failf("failed to %s nodeplugin psp: %v", action, err) - } } func validateRBDImageCount(f *framework.Framework, count int, pool string) { @@ -380,6 +339,10 @@ var _ = Describe("RBD", func() { }) Context("Test RBD CSI", func() { + if !testRBD || upgradeTesting { + return + } + It("Test RBD CSI", func() { // test only if ceph-csi is deployed via helm if helmTest { @@ -831,6 +794,11 @@ var _ = Describe("RBD", func() { baseAppName := app.Name err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { + if rwopMayFail(err) { + e2elog.Logf("RWOP is not supported: %v", err) + + return + } e2elog.Failf("failed to create PVC: %v", err) } // validate created backend rbd images @@ -865,6 +833,11 @@ var _ = Describe("RBD", func() { baseAppName := app.Name err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { + if rwopMayFail(err) { + e2elog.Logf("RWOP is not supported: %v", err) + + return + } e2elog.Failf("failed to create PVC: %v", err) } // validate created backend rbd images diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 52bcdc176..04b8427a5 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -136,15 +136,10 @@ func createRBDStorageClass( sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = rbdNodePluginSecretName - fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + fsID, err := getClusterID(f) if err != nil { - return err + return fmt.Errorf("failed to get clusterID: %w", err) } - if stdErr != "" { - return fmt.Errorf("error getting fsid %v", stdErr) - } - // remove new line present in fsID - fsID = strings.Trim(fsID, "\n") sc.Parameters["clusterID"] = fsID for k, v := range parameters { @@ -167,9 +162,22 @@ func createRBDStorageClass( sc.MountOptions = append(sc.MountOptions, mOpt...) } sc.ReclaimPolicy = &policy - _, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{}) - return err + timeout := time.Duration(deployTimeout) * time.Minute + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + _, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{}) + if err != nil { + e2elog.Logf("error creating StorageClass %q: %v", sc.Name, err) + if isRetryableAPIError(err) { + return false, nil + } + + return false, fmt.Errorf("failed to create StorageClass %q: %w", sc.Name, err) + } + + return true, nil + }) } func createRadosNamespace(f *framework.Framework) error { diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 6f2f98d3c..f0e2cdb3e 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -19,7 +19,6 @@ package e2e import ( "context" "fmt" - "strings" "time" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" @@ -158,14 +157,10 @@ func createRBDSnapshotClass(f *framework.Framework) error { sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = rbdProvisionerSecretName - fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + fsID, err := getClusterID(f) if err != nil { - return err + return fmt.Errorf("failed to get clusterID: %w", err) } - if stdErr != "" { - return fmt.Errorf("failed to get fsid from ceph cluster %s", stdErr) - } - fsID = strings.Trim(fsID, "\n") sc.Parameters["clusterID"] = fsID sclient, err := newSnapshotClient() if err != nil { @@ -193,14 +188,11 @@ func createCephFSSnapshotClass(f *framework.Framework) error { sc := getSnapshotClass(scPath) sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName - fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + + fsID, err := getClusterID(f) if err != nil { - return err + return fmt.Errorf("failed to get clusterID: %w", err) } - if stdErr != "" { - return fmt.Errorf("failed to get fsid from ceph cluster %s", stdErr) - } - fsID = strings.Trim(fsID, "\n") sc.Parameters["clusterID"] = fsID sclient, err := newSnapshotClient() if err != nil { diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index c3ad075bd..391b44fa3 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -127,15 +127,11 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI c := f.ClientSet - fsID, e, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + fsID, err := getClusterID(f) if err != nil { - return err + return fmt.Errorf("failed to get clusterID: %w", err) } - if e != "" { - return fmt.Errorf("failed to get fsid from ceph cluster %s", e) - } - // remove new line present in fsID - fsID = strings.Trim(fsID, "\n") + size := staticPVSize // create rbd image cmd := fmt.Sprintf( @@ -144,7 +140,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI staticPVSize, rbdOptions(defaultRBDPool)) - _, e, err = execCommandInToolBoxPod(f, cmd, rookNamespace) + _, e, err := execCommandInToolBoxPod(f, cmd, rookNamespace) if err != nil { return err } @@ -346,15 +342,10 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro LabelSelector: "app=rook-ceph-tools", } - fsID, e, err := execCommandInPod(f, "ceph fsid", rookNamespace, &listOpt) + fsID, err := getClusterID(f) if err != nil { - return err + return fmt.Errorf("failed to get clusterID: %w", err) } - if e != "" { - return fmt.Errorf("failed to get fsid from ceph cluster %s", e) - } - // remove new line present in fsID - fsID = strings.Trim(fsID, "\n") // 4GiB in bytes size := "4294967296" @@ -362,7 +353,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro // create subvolumegroup, command will work even if group is already present. cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fileSystemName, groupName) - _, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt) + _, e, err := execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { return err } diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index 4bd2c36bf..d74ef1f3e 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -162,6 +162,10 @@ var _ = Describe("CephFS Upgrade Testing", func() { }) Context("Cephfs Upgrade Test", func() { + if !upgradeTesting || !testCephFS { + return + } + It("Cephfs Upgrade Test", func() { By("checking provisioner deployment is running", func() { err = waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout) diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 486fce7ed..0c62ceb03 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -176,6 +176,10 @@ var _ = Describe("RBD Upgrade Testing", func() { }) Context("Test RBD CSI", func() { + if !testRBD || !upgradeTesting { + return + } + It("Test RBD CSI", func() { pvcPath := rbdExamplePath + "pvc.yaml" appPath := rbdExamplePath + "pod.yaml" diff --git a/e2e/utils.go b/e2e/utils.go index 2edcb0747..8f87acbc5 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -79,6 +79,7 @@ var ( radosNamespace string poll = 2 * time.Second isOpenShift bool + clusterID string ) func getMons(ns string, c kubernetes.Interface) ([]string, error) { @@ -121,6 +122,22 @@ func getMonsHash(mons string) string { return fmt.Sprintf("%x", md5.Sum([]byte(mons))) //nolint:gosec // hash generation } +func getClusterID(f *framework.Framework) (string, error) { + if clusterID != "" { + return clusterID, nil + } + + fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) + if err != nil { + return "", fmt.Errorf("failed getting clusterID through toolbox: %w", err) + } + if stdErr != "" { + return "", fmt.Errorf("error getting fsid: %s", stdErr) + } + // remove new line present in fsID + return strings.Trim(fsID, "\n"), nil +} + func getStorageClass(path string) (scv1.StorageClass, error) { sc := scv1.StorageClass{} err := unmarshal(path, &sc) @@ -1475,3 +1492,22 @@ func retryKubectlArgs(namespace string, action kubectlAction, t int, args ...str return true, nil }) } + +// rwopSupported indicates that a test using RWOP is expected to succeed. If +// the accessMode is reported as invalid, rwopSupported will be set to false. +var rwopSupported = true + +// rwopMayFail returns true if the accessMode is not valid. k8s v1.22 requires +// a feature gate, which might not be set. In case the accessMode is invalid, +// the featuregate is not set, and testing RWOP is not possible. +func rwopMayFail(err error) bool { + if !rwopSupported { + return true + } + + if strings.Contains(err.Error(), `invalid: spec.accessModes: Unsupported value: "ReadWriteOncePod"`) { + rwopSupported = false + } + + return !rwopSupported +} diff --git a/examples/csi-config-map-sample.yaml b/examples/csi-config-map-sample.yaml index eddab8f9a..27909d743 100644 --- a/examples/csi-config-map-sample.yaml +++ b/examples/csi-config-map-sample.yaml @@ -13,17 +13,21 @@ kind: ConfigMap # each such cluster in use. # To add more clusters or edit MON addresses in an existing configmap, use # the `kubectl replace` command. -# The is optional and represents a radosNamespace in the pool. -# If any given, all of the rbd images, snapshots, and other metadata will be -# stored within the radosNamespace. +# The "rbd.rados-namespace" is optional and represents a radosNamespace in the +# pool. If any given, all of the rbd images, snapshots, and other metadata will +# be stored within the radosNamespace. # NOTE: The given radosNamespace must already exists in the pool. # NOTE: Make sure you don't add radosNamespace option to a currently in use # configuration as it will cause issues. # The field "cephFS.subvolumeGroup" is optional and defaults to "csi". -# The fields are the various network namespace +# The "cephFS.netNamespaceFilePath" fields are the various network namespace # path for the Ceph cluster identified by the , This will be used -# by the CSI plugin to execute the rbd map/unmap and mount -t commands in the -# network namespace specified by the . +# by the CephFS CSI plugin to execute the mount -t in the +# network namespace specified by the "cephFS.netNamespaceFilePath". +# The "rbd.netNamespaceFilePath" fields are the various network namespace +# path for the Ceph cluster identified by the , This will be used +# by the RBD CSI plugin to execute the rbd map/unmap in the +# network namespace specified by the "rbd.netNamespaceFilePath". # If a CSI plugin is using more than one Ceph cluster, repeat the section for # each such cluster in use. # NOTE: Changes to the configmap is automatically updated in the running pods, @@ -42,8 +46,10 @@ data: [ { "clusterID": "", - "radosNamespace": "", - "netNamespaceFilePath": "/plugins/rbd.csi.ceph.com/net", + "rbd": { + "netNamespaceFilePath": "/plugins/rbd.csi.ceph.com/net", + "radosNamespace": "", + }, "monitors": [ "", "", @@ -52,6 +58,7 @@ data: ], "cephFS": { "subvolumeGroup": "" + "netNamespaceFilePath": "/plugins/cephfs.csi.ceph.com/net", } } ] diff --git a/go.mod b/go.mod index acfcae112..ab84a75ae 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.17 require ( github.com/IBM/keyprotect-go-client v0.7.0 - github.com/aws/aws-sdk-go v1.43.32 - github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 + github.com/aws/aws-sdk-go v1.43.41 + github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000 // TODO: API for managing NFS-exports requires `ceph_ci_untested` build-tag github.com/ceph/go-ceph v0.15.0 - github.com/container-storage-interface/spec v1.5.0 + github.com/container-storage-interface/spec v1.6.0 github.com/csi-addons/replication-lib-utils v0.2.0 github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe github.com/golang/protobuf v1.5.2 @@ -26,18 +26,18 @@ require ( github.com/stretchr/testify v1.7.1 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 - google.golang.org/grpc v1.45.0 + google.golang.org/grpc v1.46.0 google.golang.org/protobuf v1.28.0 - k8s.io/api v0.23.5 - k8s.io/apimachinery v0.23.5 + k8s.io/api v0.23.6 + k8s.io/apimachinery v0.23.6 k8s.io/client-go v12.0.0+incompatible - k8s.io/cloud-provider v0.23.5 + k8s.io/cloud-provider v0.23.6 k8s.io/klog/v2 v2.60.1 // // when updating k8s.io/kubernetes, make sure to update the replace section too // - k8s.io/kubernetes v1.23.5 - k8s.io/mount-utils v0.23.5 + k8s.io/kubernetes v1.23.6 + k8s.io/mount-utils v0.23.6 k8s.io/utils v0.0.0-20211116205334-6203023598ed sigs.k8s.io/controller-runtime v0.11.0-beta.0.0.20211208212546-f236f0345ad2 ) @@ -45,10 +45,10 @@ require ( require ( github.com/armon/go-metrics v0.3.9 // indirect github.com/armon/go-radix v1.0.0 // indirect - github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect github.com/aws/smithy-go v1.11.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect @@ -146,9 +146,9 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/apiserver v0.23.5 // indirect - k8s.io/component-base v0.23.5 // indirect - k8s.io/component-helpers v0.23.5 // indirect + k8s.io/apiserver v0.23.6 // indirect + k8s.io/component-base v0.23.6 // indirect + k8s.io/component-helpers v0.23.6 // indirect k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect k8s.io/kubectl v0.0.0 // indirect k8s.io/kubelet v0.0.0 // indirect @@ -167,31 +167,31 @@ replace ( // // k8s.io/kubernetes depends on these k8s.io packages, but unversioned // - k8s.io/api => k8s.io/api v0.23.5 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.5 - k8s.io/apimachinery => k8s.io/apimachinery v0.23.5 - k8s.io/apiserver => k8s.io/apiserver v0.23.5 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.5 - k8s.io/client-go => k8s.io/client-go v0.23.5 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.5 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.5 - k8s.io/code-generator => k8s.io/code-generator v0.23.5 - k8s.io/component-base => k8s.io/component-base v0.23.5 - k8s.io/component-helpers => k8s.io/component-helpers v0.23.5 - k8s.io/controller-manager => k8s.io/controller-manager v0.23.5 - k8s.io/cri-api => k8s.io/cri-api v0.23.5 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.5 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.5 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.5 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.5 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.5 - k8s.io/kubectl => k8s.io/kubectl v0.23.5 - k8s.io/kubelet => k8s.io/kubelet v0.23.5 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.5 - k8s.io/metrics => k8s.io/metrics v0.23.5 - k8s.io/mount-utils => k8s.io/mount-utils v0.23.5 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.5 + k8s.io/api => k8s.io/api v0.23.6 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.6 + k8s.io/apimachinery => k8s.io/apimachinery v0.23.6 + k8s.io/apiserver => k8s.io/apiserver v0.23.6 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.6 + k8s.io/client-go => k8s.io/client-go v0.23.6 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.6 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.6 + k8s.io/code-generator => k8s.io/code-generator v0.23.6 + k8s.io/component-base => k8s.io/component-base v0.23.6 + k8s.io/component-helpers => k8s.io/component-helpers v0.23.6 + k8s.io/controller-manager => k8s.io/controller-manager v0.23.6 + k8s.io/cri-api => k8s.io/cri-api v0.23.6 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.6 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.6 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.6 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.6 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.6 + k8s.io/kubectl => k8s.io/kubectl v0.23.6 + k8s.io/kubelet => k8s.io/kubelet v0.23.6 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.6 + k8s.io/metrics => k8s.io/metrics v0.23.6 + k8s.io/mount-utils => k8s.io/mount-utils v0.23.6 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.6 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.6 // layeh.com seems to be misbehaving layeh.com/radius => github.com/layeh/radius v0.0.0-20190322222518-890bc1058917 ) diff --git a/go.sum b/go.sum index aa41791d7..12aa86266 100644 --- a/go.sum +++ b/go.sum @@ -138,18 +138,18 @@ github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.43.32 h1:b2NQnfWfImfo7yzXq6gzXEC+6s5v1t2RU3G9o+VirYo= -github.com/aws/aws-sdk-go v1.43.32/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= -github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 h1:onz/VaaxZ7Z4V+WIN9Txly9XLTmoOh1oJ8XcAC3pako= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9/go.mod h1:AnVH5pvai0pAF4lXRq0bmhbes1u9R8wTE+g+183bZNM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 h1:9stUQR/u2KXU6HkFJYlqnZEjBnbgrVbG6I5HN09xZh0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3/go.mod h1:ssOhaLpRlh88H3UmEcsBoVKq309quMvm3Ds8e9d4eJM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 h1:Gh1Gpyh01Yvn7ilO/b/hr01WgNpaszfbKMUgqM186xQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3/go.mod h1:wlY6SVjuwvh3TVRpTqdy4I1JpBFLX4UGeKZdWntaocw= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 h1:cJGRyzCSVwZC7zZZ1xbx9m32UnrKydRYhOvcD1NYP9Q= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.3/go.mod h1:bfBj0iVmsUyUg4weDB4NxktD9rDGeKSVWnjTnwbx9b8= +github.com/aws/aws-sdk-go v1.43.41 h1:HaazVplP8/t6SOfybQlNUmjAxLWDKdLdX8BSEHFlJdY= +github.com/aws/aws-sdk-go v1.43.41/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go-v2 v1.16.3 h1:0W1TSJ7O6OzwuEvIXAtJGvOeQ0SGAhcpxPN2/NK5EhM= +github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 h1:uFWgo6mGJI1n17nbcvSc6fxVuR3xLNqvXt12JCnEcT8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10/go.mod h1:F+EZtuIwjlv35kRJPyBGcsA4f7bnSoz15zOQ2lJq1Z4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 h1:cnsvEKSoHN4oAN7spMMr0zhEW2MHnhAVpmqQg8E6UcM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE= github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE= github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= @@ -210,6 +210,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c/go.mod h1:XGLbWH/ujMcbPbhZq52Nv6UrCghb1yGn//133kEsvDk= @@ -218,8 +219,9 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= -github.com/container-storage-interface/spec v1.5.0 h1:lvKxe3uLgqQeVQcrnL2CPQKISoKjTJxojEs9cBk+HXo= github.com/container-storage-interface/spec v1.5.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= +github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= +github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= @@ -307,6 +309,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -450,6 +453,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -1634,8 +1638,8 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= @@ -1706,28 +1710,28 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.23.5 h1:zno3LUiMubxD/V1Zw3ijyKO3wxrhbUF1Ck+VjBvfaoA= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= -k8s.io/apiextensions-apiserver v0.23.5 h1:5SKzdXyvIJKu+zbfPc3kCbWpbxi+O+zdmAJBm26UJqI= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= -k8s.io/apimachinery v0.23.5 h1:Va7dwhp8wgkUPWsEXk6XglXWU4IKYLKNlv8VkX7SDM0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apiserver v0.23.5 h1:2Ly8oUjz5cnZRn1YwYr+aFgDZzUmEVL9RscXbnIeDSE= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= -k8s.io/cli-runtime v0.23.5/go.mod h1:oY6QDF2qo9xndSq32tqcmRp2UyXssdGrLfjAVymgbx4= -k8s.io/client-go v0.23.5 h1:zUXHmEuqx0RY4+CsnkOn5l0GU+skkRXKGJrhmE2SLd8= -k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/cloud-provider v0.23.5 h1:cf5Il2oV++RtlqgNesHd+tDFtOp85dG0t9KN/pmb71s= -k8s.io/cloud-provider v0.23.5/go.mod h1:xMZFA6pIYKweqTkWCYVgRSVMAjqOvxVr3u/kmfyxvkU= -k8s.io/cluster-bootstrap v0.23.5/go.mod h1:8/Gz6VTOMmEDDhn8U/nx0McnQR4YETAqiYXIlqR8hdQ= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.23.5 h1:8qgP5R6jG1BBSXmRYW+dsmitIrpk8F/fPEvgDenMCCE= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= -k8s.io/component-helpers v0.23.5 h1:6uTMNP6xxJrSzYTC7BCcH2S/PbSZGxSUZG0PG+nT4tM= -k8s.io/component-helpers v0.23.5/go.mod h1:5riXJgjTIs+ZB8xnf5M2anZ8iQuq37a0B/0BgoPQuSM= -k8s.io/controller-manager v0.23.5/go.mod h1:n/KRlUzAtkFcZodZ/w0GlQdmErVKh7lS/wS0bbo7W4I= -k8s.io/cri-api v0.23.5/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= -k8s.io/csi-translation-lib v0.23.5/go.mod h1:8RyFkoHAJrFU7c7MN1ZUjctm3ZhHclKm1FIHNSyGcuw= +k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw= +k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g= +k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q= +k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU= +k8s.io/apimachinery v0.23.6 h1:RH1UweWJkWNTlFx0D8uxOpaU1tjIOvVVWV/bu5b3/NQ= +k8s.io/apimachinery v0.23.6/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apiserver v0.23.6 h1:p94LiXcsSnpSDIl4cv98liBuFKcaygSCNopFNfMg/Ac= +k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg= +k8s.io/cli-runtime v0.23.6/go.mod h1:0Z3VR/HRIFKiLzKIAkm1mPtcH98GT/fXu2IU0E4vFmw= +k8s.io/client-go v0.23.6 h1:7h4SctDVQAQbkHQnR4Kzi7EyUyvla5G1pFWf4+Od7hQ= +k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4= +k8s.io/cloud-provider v0.23.6 h1:/dO0LOBiyDBV/d0tu9MjA+YCo4Bti8sy2o2jbAXI01Y= +k8s.io/cloud-provider v0.23.6/go.mod h1:kk827l5bRFlSBvSWTR0tdNzQqa44gJdqmVRjdnN+X4g= +k8s.io/cluster-bootstrap v0.23.6/go.mod h1:JbFrtiOjW1YDujFuUzPbuOSCnSUIkXUjEeci5BxB8ug= +k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8= +k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0= +k8s.io/component-helpers v0.23.6 h1:95O4LX4mk2LHWfdn/5rQJ7hOzd/teKRh9fCNXr2tJic= +k8s.io/component-helpers v0.23.6/go.mod h1:kgvl6wvnYg9oebklLPpbW8UhvAZ9Qds26/RANEbny/8= +k8s.io/controller-manager v0.23.6/go.mod h1:szOgeulY0c3xLUa9RWX1uPCjLr76RmVttghr4xQzC5U= +k8s.io/cri-api v0.23.6/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4= +k8s.io/csi-translation-lib v0.23.6/go.mod h1:8UtizkEUAnIOuTBko4/mswp3tkfMftdCEzRxJAUfxAw= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -1740,26 +1744,26 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.23.5/go.mod h1:3ynYx07Co6dzjpKPgipM+1/Mt2Jcm7dY++cRlKLr5s8= -k8s.io/kube-controller-manager v0.23.5/go.mod h1:Pkg5lIk9YG9Qjj4F7Dn0gi6/k8cEYP63oLdgrlrrtu4= +k8s.io/kube-aggregator v0.23.6/go.mod h1:cubFdoSJRMEN+ilg1ErhNIoplJwyYbmgn3bUlen8KjA= +k8s.io/kube-controller-manager v0.23.6/go.mod h1:7QZIxuo2Y1XSsCuFf+2d7EN458xWCC+0Q6f5hT0puAc= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-proxy v0.23.5/go.mod h1:/yCbRrOHgPCb1g1k4XmMJPmNesfdPhZTGrvwNlNgwo8= -k8s.io/kube-scheduler v0.23.5/go.mod h1:IJGf4WngeoAHLj4ms4n3Poa29ttmaxCXxIqpgU0ky7E= -k8s.io/kubectl v0.23.5 h1:DmDULqCaF4qstj0Im143XmncvqWtJxHzK8IrW2BzlU0= -k8s.io/kubectl v0.23.5/go.mod h1:lLgw7cVY8xbd7o637vOXPca/w6HC205KsPCRDYRCxwE= -k8s.io/kubelet v0.23.5 h1:eCGJ7olStiyF7TYHlUTjpXg2ltw7Bs9OPZcch8HP2Go= -k8s.io/kubelet v0.23.5/go.mod h1:M0aj0gaX+rOaGfCfqkV6P7QbwtMwqbL6RdwviHmnehU= -k8s.io/kubernetes v1.23.5 h1:bxpSv2BKc2MqYRfyqQqLVdodLZ2r+NZ/rEdZXyUAvug= -k8s.io/kubernetes v1.23.5/go.mod h1:avI3LUTUYZugxwh52KMVM7v9ZjB5gYJ6D3FIoZ1SHUo= -k8s.io/legacy-cloud-providers v0.23.5/go.mod h1:IENlwY686f1fbakotgNf7gAQuIyCvOUIAXkPPPE/7KU= -k8s.io/metrics v0.23.5/go.mod h1:WNAtV2a5BYbmDS8+7jSqYYV6E3efuGTpIwJ8PTD1wgs= -k8s.io/mount-utils v0.23.5 h1:MOhJKZTpfC21r5OamKYWMdVNtTMDD9wZfTkLOhI5nuE= -k8s.io/mount-utils v0.23.5/go.mod h1:OTN3LQPiOGMfx/SmVlsnySwsAmh4gYrDYLchlMHtf98= -k8s.io/pod-security-admission v0.23.5/go.mod h1:aSyWfjev8Zil5DaZBZ+ICAObZmZlRqhnAZHxA9r71UI= -k8s.io/sample-apiserver v0.23.5/go.mod h1:m4cnT3HgRY5Dt2AjMVKGnb31D6rGY0B+xpKtRJUUC8w= +k8s.io/kube-proxy v0.23.6/go.mod h1:HyB7KVZ88iRtKkrr3mFPIL+k5tZZObk/2MjHl41Po68= +k8s.io/kube-scheduler v0.23.6/go.mod h1:BqgJohUEgt2Fsg9cBipnyDiaPgexodxivrbNEACcsqQ= +k8s.io/kubectl v0.23.6 h1:ajzrqj88GqlH/gpscMCts+mKNeSJprpkWJEHO8CR2Ss= +k8s.io/kubectl v0.23.6/go.mod h1:mMtJhc2QtQiSfvIQoMEUIjGHtZuP4uxMy/ees6j6zx8= +k8s.io/kubelet v0.23.6 h1:tuscMqYCt9cxWursmTU9OJ2tPLv66Ji+AGbuV1Z/lug= +k8s.io/kubelet v0.23.6/go.mod h1:ROttmKIUkB9in4NyX/SfnAoXGfW/Dju3VCGFP34F5ac= +k8s.io/kubernetes v1.23.6 h1:IGoo8aUBuIep0G4NW5ezejuH16sMX0zd2sx/bD+D+Z0= +k8s.io/kubernetes v1.23.6/go.mod h1:avI3LUTUYZugxwh52KMVM7v9ZjB5gYJ6D3FIoZ1SHUo= +k8s.io/legacy-cloud-providers v0.23.6/go.mod h1:jc5asdak2B4UW77dOWTTK8XvJak+MxyXdCHr4r/M7mA= +k8s.io/metrics v0.23.6/go.mod h1:Fm9VzVMZ7KVEEeLStF2y3XogfcDwpGyI15o1xB6PbYk= +k8s.io/mount-utils v0.23.6 h1:NTLirnhIWsv/UtBYLNpwtftBPeoAAwPocyx0B2KsCxs= +k8s.io/mount-utils v0.23.6/go.mod h1:OTN3LQPiOGMfx/SmVlsnySwsAmh4gYrDYLchlMHtf98= +k8s.io/pod-security-admission v0.23.6/go.mod h1:molyEVwa8EtBxQ/LgizUHnN8l0su2PQtLSROnF3P9Js= +k8s.io/sample-apiserver v0.23.6/go.mod h1:32pvunKq7Px1NZsIxGwQKy8o+i4FfK/2rCL/htr7Uig= k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q= k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/internal/cephfs/nodeserver.go b/internal/cephfs/nodeserver.go index b7c10db55..fdadb6497 100644 --- a/internal/cephfs/nodeserver.go +++ b/internal/cephfs/nodeserver.go @@ -126,11 +126,12 @@ func (ns *NodeServer) NodeStageVolume( } defer volOptions.Destroy() - volOptions.NetNamespaceFilePath, err = util.GetNetNamespaceFilePath(util.CsiConfigFile, volOptions.ClusterID) + volOptions.NetNamespaceFilePath, err = util.GetCephFSNetNamespaceFilePath( + util.CsiConfigFile, + volOptions.ClusterID) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } - mnt, err := mounter.New(volOptions) if err != nil { log.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err) diff --git a/internal/csi-addons/rbd/network_fence_test.go b/internal/csi-addons/rbd/network_fence_test.go index 8384e4b3d..80ab82130 100644 --- a/internal/csi-addons/rbd/network_fence_test.go +++ b/internal/csi-addons/rbd/network_fence_test.go @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" - fence "github.com/csi-addons/spec/lib/go/fence" + "github.com/csi-addons/spec/lib/go/fence" ) // TestFenceClusterNetwork is a minimal test for the FenceClusterNetwork() diff --git a/internal/nfs/controller/controllerserver.go b/internal/nfs/controller/controllerserver.go index c49c95d7a..75a0c515c 100644 --- a/internal/nfs/controller/controllerserver.go +++ b/internal/nfs/controller/controllerserver.go @@ -18,7 +18,6 @@ package controller import ( "context" - "fmt" "github.com/ceph/ceph-csi/internal/cephfs" "github.com/ceph/ceph-csi/internal/cephfs/store" @@ -75,7 +74,7 @@ func (cs *Server) CreateVolume( req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { res, err := cs.backendServer.CreateVolume(ctx, req) if err != nil { - return res, fmt.Errorf("failed to create CephFS volume: %w", err) + return nil, err } backend := res.Volume diff --git a/internal/rbd/mirror.go b/internal/rbd/mirror.go index 08a5e52b8..239c81667 100644 --- a/internal/rbd/mirror.go +++ b/internal/rbd/mirror.go @@ -148,7 +148,7 @@ func (ri *rbdImage) resyncImage() error { return nil } -// getImageMirroingStatus get the mirroring status of an image. +// getImageMirroringStatus get the mirroring status of an image. func (ri *rbdImage) getImageMirroringStatus() (*librbd.GlobalMirrorImageStatus, error) { image, err := ri.open() if err != nil { diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 579f2c211..3ac153f44 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -332,7 +332,7 @@ func (ns *NodeServer) NodeStageVolume( } defer rv.Destroy() - rv.NetNamespaceFilePath, err = util.GetNetNamespaceFilePath(util.CsiConfigFile, rv.ClusterID) + rv.NetNamespaceFilePath, err = util.GetRBDNetNamespaceFilePath(util.CsiConfigFile, rv.ClusterID) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } diff --git a/internal/rbd/rbd_journal.go b/internal/rbd/rbd_journal.go index a8907ac7e..dba2ddab7 100644 --- a/internal/rbd/rbd_journal.go +++ b/internal/rbd/rbd_journal.go @@ -524,7 +524,7 @@ func undoVolReservation(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent // complete omap mapping between imageName and volumeID. // RegenerateJournal performs below operations -// Extract clusterID, Mons after checkig clusterID mapping +// Extract clusterID, Mons after checking clusterID mapping // Extract parameters journalPool, pool from volumeAttributes // Extract optional parameters volumeNamePrefix, kmsID, owner from volumeAttributes // Extract information from volumeID diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 7f00cfe18..a7c5804e3 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -942,7 +942,7 @@ func genSnapFromSnapID( return err } -// updateSnapshotDetails will copies the details from the rbdVolume to the +// updateSnapshotDetails will copy the details from the rbdVolume to the // rbdSnapshot. example copying size from rbdVolume to rbdSnapshot. func updateSnapshotDetails(rbdSnap *rbdSnapshot) error { vol := generateVolFromSnap(rbdSnap) diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index edeb0c0d3..51068daed 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -464,7 +464,7 @@ func disableVolumeReplication(rbdVol *rbdVolume, // Local image is in up+replaying state // If the image is in a secondary and its state is up+replaying means - // its an healthy secondary and the image is primary somewhere in the + // its a healthy secondary and the image is primary somewhere in the // remote cluster and the local image is getting replayed. Return // success for the Disabling mirroring as we cannot disable mirroring // on the secondary image, when the image on the primary site gets diff --git a/internal/util/cephcmds.go b/internal/util/cephcmds.go index 71999e98d..924024d46 100644 --- a/internal/util/cephcmds.go +++ b/internal/util/cephcmds.go @@ -38,9 +38,9 @@ const InvalidPoolID int64 = -1 // context.TODO(), the command will be logged after it was executed. func ExecuteCommandWithNSEnter(ctx context.Context, netPath, program string, args ...string) (string, string, error) { var ( - sanitizedArgs = StripSecretInArgs(args) - stdoutBuf bytes.Buffer - stderrBuf bytes.Buffer + stdoutBuf bytes.Buffer + stderrBuf bytes.Buffer + nsenter = "nsenter" ) // check netPath exists @@ -49,8 +49,8 @@ func ExecuteCommandWithNSEnter(ctx context.Context, netPath, program string, arg } // nsenter --net=%s -- args = append([]string{fmt.Sprintf("--net=%s", netPath), "--", program}, args...) - cmd := exec.Command("nsenter", args...) // #nosec:G204, commands executing not vulnerable. - + sanitizedArgs := StripSecretInArgs(args) + cmd := exec.Command(nsenter, args...) // #nosec:G204, commands executing not vulnerable. cmd.Stdout = &stdoutBuf cmd.Stderr = &stderrBuf @@ -59,7 +59,7 @@ func ExecuteCommandWithNSEnter(ctx context.Context, netPath, program string, arg stderr := stderrBuf.String() if err != nil { - err = fmt.Errorf("an error (%w) occurred while running %s args: %v", err, program, sanitizedArgs) + err = fmt.Errorf("an error (%w) occurred while running %s args: %v", err, nsenter, sanitizedArgs) if ctx != context.TODO() { log.UsefulLog(ctx, "%s", err) } @@ -68,7 +68,7 @@ func ExecuteCommandWithNSEnter(ctx context.Context, netPath, program string, arg } if ctx != context.TODO() { - log.UsefulLog(ctx, "command succeeded: %s %v", program, sanitizedArgs) + log.UsefulLog(ctx, "command succeeded: %s %v", nsenter, sanitizedArgs) } return stdout, stderr, nil diff --git a/internal/util/csiconfig.go b/internal/util/csiconfig.go index a6e3dd534..bdfe6d267 100644 --- a/internal/util/csiconfig.go +++ b/internal/util/csiconfig.go @@ -41,35 +41,43 @@ type ClusterInfo struct { // ClusterID is used for unique identification ClusterID string `json:"clusterID"` // RadosNamespace is a rados namespace in the pool - RadosNamespace string `json:"radosNamespace"` + RadosNamespace string `json:"radosNamespace"` // For backward compatibility. TODO: Remove this in 3.7.0 // Monitors is monitor list for corresponding cluster ID Monitors []string `json:"monitors"` // CephFS contains CephFS specific options CephFS struct { + // symlink filepath for the network namespace where we need to execute commands. + NetNamespaceFilePath string `json:"netNamespaceFilePath"` // SubvolumeGroup contains the name of the SubvolumeGroup for CSI volumes SubvolumeGroup string `json:"subvolumeGroup"` } `json:"cephFS"` - // symlink filepath for the network namespace where we need to execute commands. - NetNamespaceFilePath string `json:"netNamespaceFilePath"` + + // RBD Contains RBD specific options + RBD struct { + // symlink filepath for the network namespace where we need to execute commands. + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + // RadosNamespace is a rados namespace in the pool + RadosNamespace string `json:"radosNamespace"` + } `json:"rbd"` } // Expected JSON structure in the passed in config file is, -// [ -// { -// "clusterID": "", -// "radosNamespace": "", -// "monitors": -// [ -// "", -// "", -// ... -// ], -// "cephFS": { -// "subvolumeGroup": "" -// } -// }, -// ... -// ]. +// nolint:godot // example json content should not contain unwanted dot. +/* +[{ + "clusterID": "", + "rbd": { + "radosNamespace": "" + }, + "monitors": [ + "", + "" + ], + "cephFS": { + "subvolumeGroup": "" + } +}] +*/ func readClusterInfo(pathToConfig, clusterID string) (*ClusterInfo, error) { var config []ClusterInfo @@ -117,6 +125,10 @@ func GetRadosNamespace(pathToConfig, clusterID string) (string, error) { return "", err } + if cluster.RBD.RadosNamespace != "" { + return cluster.RBD.RadosNamespace, nil + } + return cluster.RadosNamespace, nil } @@ -164,11 +176,21 @@ func GetClusterID(options map[string]string) (string, error) { return clusterID, nil } -func GetNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) { +func GetRBDNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) { cluster, err := readClusterInfo(pathToConfig, clusterID) if err != nil { return "", err } - return cluster.NetNamespaceFilePath, nil + return cluster.RBD.NetNamespaceFilePath, nil +} + +// GetCephFSNetNamespaceFilePath returns the netNamespaceFilePath for CephFS volumes. +func GetCephFSNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) { + cluster, err := readClusterInfo(pathToConfig, clusterID) + if err != nil { + return "", err + } + + return cluster.CephFS.NetNamespaceFilePath, nil } diff --git a/internal/util/csiconfig_test.go b/internal/util/csiconfig_test.go index 22ceadef2..4a37e588b 100644 --- a/internal/util/csiconfig_test.go +++ b/internal/util/csiconfig_test.go @@ -140,7 +140,7 @@ func TestCSIConfig(t *testing.T) { } } -func TestGetNetNamespaceFilePath(t *testing.T) { +func TestGetRBDNetNamespaceFilePath(t *testing.T) { t.Parallel() tests := []struct { name string @@ -148,17 +148,17 @@ func TestGetNetNamespaceFilePath(t *testing.T) { want string }{ { - name: "get NetNamespaceFilePath for cluster-1", + name: "get RBD NetNamespaceFilePath for cluster-1", clusterID: "cluster-1", want: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster1-net", }, { - name: "get NetNamespaceFilePath for cluster-2", + name: "get RBD NetNamespaceFilePath for cluster-2", clusterID: "cluster-2", want: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster2-net", }, { - name: "when NetNamespaceFilePath is empty", + name: "when RBD NetNamespaceFilePath is empty", clusterID: "cluster-3", want: "", }, @@ -166,14 +166,24 @@ func TestGetNetNamespaceFilePath(t *testing.T) { csiConfig := []ClusterInfo{ { - ClusterID: "cluster-1", - Monitors: []string{"ip-1", "ip-2"}, - NetNamespaceFilePath: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster1-net", + ClusterID: "cluster-1", + Monitors: []string{"ip-1", "ip-2"}, + RBD: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + RadosNamespace string `json:"radosNamespace"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster1-net", + }, }, { - ClusterID: "cluster-2", - Monitors: []string{"ip-3", "ip-4"}, - NetNamespaceFilePath: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster2-net", + ClusterID: "cluster-2", + Monitors: []string{"ip-3", "ip-4"}, + RBD: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + RadosNamespace string `json:"radosNamespace"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/rbd.ceph.csi.com/cluster2-net", + }, }, { ClusterID: "cluster-3", @@ -193,14 +203,90 @@ func TestGetNetNamespaceFilePath(t *testing.T) { ts := tt t.Run(ts.name, func(t *testing.T) { t.Parallel() - got, err := GetNetNamespaceFilePath(tmpConfPath, ts.clusterID) + got, err := GetRBDNetNamespaceFilePath(tmpConfPath, ts.clusterID) if err != nil { - t.Errorf("GetNetNamespaceFilePath() error = %v", err) + t.Errorf("GetRBDNetNamespaceFilePath() error = %v", err) return } if got != ts.want { - t.Errorf("GetNetNamespaceFilePath() = %v, want %v", got, ts.want) + t.Errorf("GetRBDNetNamespaceFilePath() = %v, want %v", got, ts.want) + } + }) + } +} + +func TestGetCephFSNetNamespaceFilePath(t *testing.T) { + t.Parallel() + tests := []struct { + name string + clusterID string + want string + }{ + { + name: "get cephFS specific NetNamespaceFilePath for cluster-1", + clusterID: "cluster-1", + want: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster1-net", + }, + { + name: "get cephFS specific NetNamespaceFilePath for cluster-2", + clusterID: "cluster-2", + want: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster2-net", + }, + { + name: "when cephFS specific NetNamespaceFilePath is empty", + clusterID: "cluster-3", + want: "", + }, + } + + csiConfig := []ClusterInfo{ + { + ClusterID: "cluster-1", + Monitors: []string{"ip-1", "ip-2"}, + CephFS: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + SubvolumeGroup string `json:"subvolumeGroup"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster1-net", + }, + }, + { + ClusterID: "cluster-2", + Monitors: []string{"ip-3", "ip-4"}, + CephFS: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + SubvolumeGroup string `json:"subvolumeGroup"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster2-net", + }, + }, + { + ClusterID: "cluster-3", + Monitors: []string{"ip-5", "ip-6"}, + }, + } + csiConfigFileContent, err := json.Marshal(csiConfig) + if err != nil { + t.Errorf("failed to marshal csi config info %v", err) + } + tmpConfPath := t.TempDir() + "/ceph-csi.json" + err = os.WriteFile(tmpConfPath, csiConfigFileContent, 0o600) + if err != nil { + t.Errorf("failed to write %s file content: %v", CsiConfigFile, err) + } + for _, tt := range tests { + ts := tt + t.Run(ts.name, func(t *testing.T) { + t.Parallel() + got, err := GetCephFSNetNamespaceFilePath(tmpConfPath, ts.clusterID) + if err != nil { + t.Errorf("GetCephFSNetNamespaceFilePath() error = %v", err) + + return + } + if got != ts.want { + t.Errorf("GetCephFSNetNamespaceFilePath() = %v, want %v", got, ts.want) } }) } diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 549dbc35c..127ddf891 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -162,7 +162,7 @@ NUM_DISKS=${NUM_DISKS:-"1"} DISK_SIZE=${DISK_SIZE:-"32g"} #configure image repo CEPHCSI_IMAGE_REPO=${CEPHCSI_IMAGE_REPO:-"quay.io/cephcsi"} -K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"k8s.gcr.io/sig-storage"} +K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"registry.k8s.io/sig-storage"} DISK="sda1" if [[ "${VM_DRIVER}" == "kvm2" ]]; then # use vda1 instead of sda1 when running with the libvirt driver diff --git a/scripts/utils.sh b/scripts/utils.sh index a7605b7c3..d89acaffe 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -3,6 +3,20 @@ KUBECTL_RETRY=5 KUBECTL_RETRY_DELAY=10 +# kubectl_retry calls `kubectl` with the passed arguments. In case of a +# failure, the `kubectl` command will be retried for `KUBECTL_RETRY` times, +# with a delay of `KUBECTL_RETRY_DELAY` between them. +# +# Upon creation failures, `AlreadyExists` and `Warning` are ignored, making +# sure the create succeeds in case some objects were created successfully in a +# previous try. +# +# Upon deletion failures, the same applies as for creation, except that +# NotFound is ignored. +# +# Logs from `kubectl` are passed on to stdout, so that a calling function can +# capture it. During the function, logs are written to stderr as to not +# interfere with the log parsing of the calling function. kubectl_retry() { local retries=0 action="${1}" ret=0 stdout stderr shift @@ -11,13 +25,13 @@ kubectl_retry() { stdout=$(mktemp rook-kubectl-stdout.XXXXXXXX) stderr=$(mktemp rook-kubectl-stderr.XXXXXXXX) - while ! kubectl "${action}" "${@}" 2>"${stderr}" 1>"${stdout}" + while ! ( kubectl "${action}" "${@}" 2>"${stderr}" 1>>"${stdout}" ) do # in case of a failure when running "create", ignore errors with "AlreadyExists" if [ "${action}" == 'create' ] then - # count lines in stderr that do not have "AlreadyExists" - ret=$(grep -cvw 'AlreadyExists' "${stderr}") + # count lines in stderr that do not have "AlreadyExists" or "Warning" + ret=$(grep -cvw -e 'AlreadyExists' -e '^Warning:' "${stderr}") if [ "${ret}" -eq 0 ] then # Success! stderr is empty after removing all "AlreadyExists" lines. @@ -28,8 +42,8 @@ kubectl_retry() { # in case of a failure when running "delete", ignore errors with "NotFound" if [ "${action}" == 'delete' ] then - # count lines in stderr that do not have "NotFound" - ret=$(grep -cvw 'NotFound' "${stderr}") + # count lines in stderr that do not have "NotFound" or "Warning" + ret=$(grep -cvw -e 'NotFound' -e '^Warning:' "${stderr}") if [ "${ret}" -eq 0 ] then # Success! stderr is empty after removing all "NotFound" lines. @@ -44,10 +58,11 @@ kubectl_retry() { break fi - # log stderr and empty the tmpfile + # write logs to stderr and empty stderr (only) + cat "${stdout}" > /dev/stderr cat "${stderr}" > /dev/stderr true > "${stderr}" - echo "kubectl_retry ${*} failed, will retry in ${KUBECTL_RETRY_DELAY} seconds" + echo "$(date): 'kubectl_retry ${*}' failed (${retries}/${KUBECTL_RETRY}), will retry in ${KUBECTL_RETRY_DELAY} seconds" > /dev/stderr sleep ${KUBECTL_RETRY_DELAY} @@ -56,6 +71,8 @@ kubectl_retry() { ret=0 done + echo "$(date): 'kubectl_retry ${*}' done (ret=${ret})" > /dev/stderr + # write output so that calling functions can consume it cat "${stdout}" > /dev/stdout cat "${stderr}" > /dev/stderr diff --git a/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go index f12e3356f..8dd3973bf 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go @@ -3,4 +3,4 @@ package aws // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.16.2" +const goModuleVersion = "1.16.3" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/CHANGELOG.md b/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/CHANGELOG.md index 856aec912..3ab362b42 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/CHANGELOG.md +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.1.10 (2022-04-25) + +* **Dependency Update**: Updated to the latest SDK module versions + # v1.1.9 (2022-03-30) * **Dependency Update**: Updated to the latest SDK module versions diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/go_module_metadata.go index d17db4207..23ed79f51 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/go_module_metadata.go @@ -3,4 +3,4 @@ package configsources // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.1.9" +const goModuleVersion = "1.1.10" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/CHANGELOG.md b/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/CHANGELOG.md index 3473a9823..ff627acb0 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/CHANGELOG.md +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/CHANGELOG.md @@ -1,3 +1,7 @@ +# v2.4.4 (2022-04-25) + +* **Dependency Update**: Updated to the latest SDK module versions + # v2.4.3 (2022-03-30) * **Dependency Update**: Updated to the latest SDK module versions diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/go_module_metadata.go index 88f74540a..79bdc8efd 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/go_module_metadata.go @@ -3,4 +3,4 @@ package endpoints // goModuleVersion is the tagged release for this module -const goModuleVersion = "2.4.3" +const goModuleVersion = "2.4.4" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/LICENSE b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/LICENSE index 6a66aea5e..fe6a62006 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/LICENSE +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/LICENSE @@ -14,7 +14,7 @@ distribution. contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +THIS SOFTWARE IS PROVIDED BY THE COPYIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT @@ -25,3 +25,4 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/docs.go b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/docs.go new file mode 100644 index 000000000..cb70616e8 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/docs.go @@ -0,0 +1,7 @@ +// Package singleflight provides a duplicate function call suppression +// mechanism. This package is a fork of the Go golang.org/x/sync/singleflight +// package. The package is forked, because the package a part of the unstable +// and unversioned golang.org/x/sync module. +// +// https://github.com/golang/sync/tree/67f06af15bc961c363a7260195bcd53487529a21/singleflight +package singleflight diff --git a/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/singleflight.go b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/singleflight.go index 14ad0c589..e8a1b17d5 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/singleflight.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/internal/sync/singleflight/singleflight.go @@ -2,11 +2,44 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package singleflight provides a duplicate function call suppression -// mechanism. package singleflight -import "sync" +import ( + "bytes" + "errors" + "fmt" + "runtime" + "runtime/debug" + "sync" +) + +// errGoexit indicates the runtime.Goexit was called in +// the user given function. +var errGoexit = errors.New("runtime.Goexit was called") + +// A panicError is an arbitrary value recovered from a panic +// with the stack trace during the execution of given function. +type panicError struct { + value interface{} + stack []byte +} + +// Error implements error interface. +func (p *panicError) Error() string { + return fmt.Sprintf("%v\n\n%s", p.value, p.stack) +} + +func newPanicError(v interface{}) error { + stack := debug.Stack() + + // The first line of the stack trace is of the form "goroutine N [status]:" + // but by the time the panic reaches Do the goroutine may no longer exist + // and its status will have changed. Trim out the misleading line. + if line := bytes.IndexByte(stack[:], '\n'); line >= 0 { + stack = stack[line+1:] + } + return &panicError{value: v, stack: stack} +} // call is an in-flight or completed singleflight.Do call type call struct { @@ -57,6 +90,12 @@ func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, e c.dups++ g.mu.Unlock() c.wg.Wait() + + if e, ok := c.err.(*panicError); ok { + panic(e) + } else if c.err == errGoexit { + runtime.Goexit() + } return c.val, c.err, true } c := new(call) @@ -70,6 +109,8 @@ func (g *Group) Do(key string, fn func() (interface{}, error)) (v interface{}, e // DoChan is like Do but returns a channel that will receive the // results when they are ready. +// +// The returned channel will not be closed. func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result { ch := make(chan Result, 1) g.mu.Lock() @@ -94,17 +135,66 @@ func (g *Group) DoChan(key string, fn func() (interface{}, error)) <-chan Result // doCall handles the single call for a key. func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { - c.val, c.err = fn() - c.wg.Done() + normalReturn := false + recovered := false - g.mu.Lock() - if !c.forgotten { - delete(g.m, key) + // use double-defer to distinguish panic from runtime.Goexit, + // more details see https://golang.org/cl/134395 + defer func() { + // the given function invoked runtime.Goexit + if !normalReturn && !recovered { + c.err = errGoexit + } + + c.wg.Done() + g.mu.Lock() + defer g.mu.Unlock() + if !c.forgotten { + delete(g.m, key) + } + + if e, ok := c.err.(*panicError); ok { + // In order to prevent the waiting channels from being blocked forever, + // needs to ensure that this panic cannot be recovered. + if len(c.chans) > 0 { + go panic(e) + select {} // Keep this goroutine around so that it will appear in the crash dump. + } else { + panic(e) + } + } else if c.err == errGoexit { + // Already in the process of goexit, no need to call again + } else { + // Normal return + for _, ch := range c.chans { + ch <- Result{c.val, c.err, c.dups > 0} + } + } + }() + + func() { + defer func() { + if !normalReturn { + // Ideally, we would wait to take a stack trace until we've determined + // whether this is a panic or a runtime.Goexit. + // + // Unfortunately, the only way we can distinguish the two is to see + // whether the recover stopped the goroutine from terminating, and by + // the time we know that, the part of the stack trace relevant to the + // panic has been discarded. + if r := recover(); r != nil { + c.err = newPanicError(r) + } + } + }() + + c.val, c.err = fn() + normalReturn = true + }() + + if !normalReturn { + recovered = true } - for _, ch := range c.chans { - ch <- Result{c.val, c.err, c.dups > 0} - } - g.mu.Unlock() } // Forget tells the singleflight to forget about a key. Future calls diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/CHANGELOG.md b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/CHANGELOG.md index d779c398d..ebcc6a461 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/CHANGELOG.md +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.9.4 (2022-04-25) + +* **Dependency Update**: Updated to the latest SDK module versions + # v1.9.3 (2022-03-30) * **Dependency Update**: Updated to the latest SDK module versions diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go index f30bfff65..a312a9efa 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go @@ -3,4 +3,4 @@ package presignedurl // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.9.3" +const goModuleVersion = "1.9.4" diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md b/vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md index 6ad12851a..7c861f7cd 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md @@ -1,3 +1,7 @@ +# v1.16.4 (2022-04-25) + +* **Dependency Update**: Updated to the latest SDK module versions + # v1.16.3 (2022-03-30) * **Dependency Update**: Updated to the latest SDK module versions diff --git a/vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go b/vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go index c81e72b70..e55b7a74a 100644 --- a/vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go +++ b/vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go @@ -3,4 +3,4 @@ package sts // goModuleVersion is the tagged release for this module -const goModuleVersion = "1.16.3" +const goModuleVersion = "1.16.4" diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go index e3208828b..21fb2c191 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go @@ -11160,12 +11160,42 @@ var awsPartition = partition{ endpointKey{ Region: "eu-west-1", }: endpoint{}, + endpointKey{ + Region: "fips-us-east-1", + }: endpoint{ + Hostname: "iotsitewise-fips.us-east-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-east-1", + }, + Deprecated: boxedTrue, + }, + endpointKey{ + Region: "fips-us-west-2", + }: endpoint{ + Hostname: "iotsitewise-fips.us-west-2.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-west-2", + }, + Deprecated: boxedTrue, + }, endpointKey{ Region: "us-east-1", }: endpoint{}, + endpointKey{ + Region: "us-east-1", + Variant: fipsVariant, + }: endpoint{ + Hostname: "iotsitewise-fips.us-east-1.amazonaws.com", + }, endpointKey{ Region: "us-west-2", }: endpoint{}, + endpointKey{ + Region: "us-west-2", + Variant: fipsVariant, + }: endpoint{ + Hostname: "iotsitewise-fips.us-west-2.amazonaws.com", + }, }, }, "iotthingsgraph": service{ @@ -15477,6 +15507,9 @@ var awsPartition = partition{ endpointKey{ Region: "ap-southeast-2", }: endpoint{}, + endpointKey{ + Region: "ap-southeast-3", + }: endpoint{}, endpointKey{ Region: "ca-central-1", }: endpoint{}, @@ -18518,6 +18551,9 @@ var awsPartition = partition{ endpointKey{ Region: "ap-northeast-2", }: endpoint{}, + endpointKey{ + Region: "ap-northeast-3", + }: endpoint{}, endpointKey{ Region: "ap-south-1", }: endpoint{}, @@ -18527,6 +18563,9 @@ var awsPartition = partition{ endpointKey{ Region: "ap-southeast-2", }: endpoint{}, + endpointKey{ + Region: "ap-southeast-3", + }: endpoint{}, endpointKey{ Region: "ca-central-1", }: endpoint{}, @@ -19128,6 +19167,40 @@ var awsPartition = partition{ }, }, }, + "sms-voice": service{ + Endpoints: serviceEndpoints{ + endpointKey{ + Region: "ap-northeast-1", + }: endpoint{}, + endpointKey{ + Region: "ap-south-1", + }: endpoint{}, + endpointKey{ + Region: "ap-southeast-1", + }: endpoint{}, + endpointKey{ + Region: "ap-southeast-2", + }: endpoint{}, + endpointKey{ + Region: "ca-central-1", + }: endpoint{}, + endpointKey{ + Region: "eu-central-1", + }: endpoint{}, + endpointKey{ + Region: "eu-west-1", + }: endpoint{}, + endpointKey{ + Region: "eu-west-2", + }: endpoint{}, + endpointKey{ + Region: "us-east-1", + }: endpoint{}, + endpointKey{ + Region: "us-west-2", + }: endpoint{}, + }, + }, "snowball": service{ Endpoints: serviceEndpoints{ endpointKey{ @@ -26394,9 +26467,24 @@ var awsusgovPartition = partition{ }, "iotsitewise": service{ Endpoints: serviceEndpoints{ + endpointKey{ + Region: "fips-us-gov-west-1", + }: endpoint{ + Hostname: "iotsitewise-fips.us-gov-west-1.amazonaws.com", + CredentialScope: credentialScope{ + Region: "us-gov-west-1", + }, + Deprecated: boxedTrue, + }, endpointKey{ Region: "us-gov-west-1", }: endpoint{}, + endpointKey{ + Region: "us-gov-west-1", + Variant: fipsVariant, + }: endpoint{ + Hostname: "iotsitewise-fips.us-gov-west-1.amazonaws.com", + }, }, }, "kafka": service{ @@ -27981,6 +28069,13 @@ var awsusgovPartition = partition{ }, }, }, + "sms-voice": service{ + Endpoints: serviceEndpoints{ + endpointKey{ + Region: "us-gov-west-1", + }: endpoint{}, + }, + }, "snowball": service{ Endpoints: serviceEndpoints{ endpointKey{ @@ -28983,6 +29078,18 @@ var awsisoPartition = partition{ }: endpoint{}, }, }, + "eks": service{ + Defaults: endpointDefaults{ + defaultKey{}: endpoint{ + Protocols: []string{"http", "https"}, + }, + }, + Endpoints: serviceEndpoints{ + endpointKey{ + Region: "us-iso-east-1", + }: endpoint{}, + }, + }, "elasticache": service{ Endpoints: serviceEndpoints{ endpointKey{ @@ -29652,6 +29759,18 @@ var awsisobPartition = partition{ }: endpoint{}, }, }, + "eks": service{ + Defaults: endpointDefaults{ + defaultKey{}: endpoint{ + Protocols: []string{"http", "https"}, + }, + }, + Endpoints: serviceEndpoints{ + endpointKey{ + Region: "us-isob-east-1", + }: endpoint{}, + }, + }, "elasticache": service{ Endpoints: serviceEndpoints{ endpointKey{ diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go index 2b141994c..88cfe6f5b 100644 --- a/vendor/github.com/aws/aws-sdk-go/aws/version.go +++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go @@ -5,4 +5,4 @@ package aws const SDKName = "aws-sdk-go" // SDKVersion is the version of this SDK -const SDKVersion = "1.43.32" +const SDKVersion = "1.43.41" diff --git a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go index e2df85864..85508634c 100644 --- a/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go +++ b/vendor/github.com/aws/aws-sdk-go/service/ec2/api.go @@ -4339,26 +4339,14 @@ func (c *EC2) CreateCustomerGatewayRequest(input *CreateCustomerGatewayInput) (r // For devices that use Border Gateway Protocol (BGP), you can also provide // the device's BGP Autonomous System Number (ASN). You can use an existing // ASN assigned to your network. If you don't have an ASN already, you can use -// a private ASN (in the 64512 - 65534 range). -// -// Amazon EC2 supports all 4-byte ASN numbers in the range of 1 - 2147483647, -// with the exception of the following: -// -// * 7224 - reserved in the us-east-1 Region -// -// * 9059 - reserved in the eu-west-1 Region -// -// * 17943 - reserved in the ap-southeast-1 Region -// -// * 10124 - reserved in the ap-northeast-1 Region -// -// For more information, see Amazon Web Services Site-to-Site VPN (https://docs.aws.amazon.com/vpn/latest/s2svpn/VPC_VPN.html) +// a private ASN. For more information, see Customer gateway options for your +// Site-to-Site VPN connection (https://docs.aws.amazon.com/vpn/latest/s2svpn/cgw-options.html) // in the Amazon Web Services Site-to-Site VPN User Guide. // // To create more than one customer gateway with the same VPN type, IP address, -// and BGP ASN, specify a unique device name for each customer gateway. Identical -// requests return information about the existing customer gateway and do not -// create new customer gateways. +// and BGP ASN, specify a unique device name for each customer gateway. An identical +// request returns information about the existing customer gateway; it doesn't +// create a new customer gateway. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -5039,11 +5027,15 @@ func (c *EC2) CreateImageRequest(input *CreateImageInput) (req *request.Request, // Creates an Amazon EBS-backed AMI from an Amazon EBS-backed instance that // is either running or stopped. // -// By default, Amazon EC2 shuts down and reboots the instance before creating -// the AMI to ensure that everything on the instance is stopped and in a consistent -// state during the creation process. If you're confident that your instance -// is in a consistent state appropriate for AMI creation, use the NoReboot parameter -// to prevent Amazon EC2 from shutting down and rebooting the instance. +// By default, when Amazon EC2 creates the new AMI, it reboots the instance +// so that it can take snapshots of the attached volumes while data is at rest, +// in order to ensure a consistent state. You can set the NoReboot parameter +// to true in the API request, or use the --no-reboot option in the CLI to prevent +// Amazon EC2 from shutting down and rebooting the instance. +// +// If you choose to bypass the shutdown and reboot process by setting the NoReboot +// parameter to true in the API request, or by using the --no-reboot option +// in the CLI, we can't guarantee the file system integrity of the created image. // // If you customized your instance with instance store volumes or Amazon EBS // volumes in addition to the root device volume, the new AMI contains block @@ -5380,7 +5372,7 @@ func (c *EC2) CreateIpamRequest(input *CreateIpamInput) (req *request.Request, o // CreateIpam API operation for Amazon Elastic Compute Cloud. // -// Create an IPAM. Amazon VCP IP Address Manager (IPAM) is a VPC feature that +// Create an IPAM. Amazon VPC IP Address Manager (IPAM) is a VPC feature that // you can use to automate your IP address management workflows including assigning, // tracking, troubleshooting, and auditing IP addresses across Amazon Web Services // Regions and accounts throughout your Amazon Web Services Organization. @@ -5710,10 +5702,18 @@ func (c *EC2) CreateLaunchTemplateRequest(input *CreateLaunchTemplateInput) (req // CreateLaunchTemplate API operation for Amazon Elastic Compute Cloud. // -// Creates a launch template. A launch template contains the parameters to launch -// an instance. When you launch an instance using RunInstances, you can specify -// a launch template instead of providing the launch parameters in the request. -// For more information, see Launching an instance from a launch template (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html) +// Creates a launch template. +// +// A launch template contains the parameters to launch an instance. When you +// launch an instance using RunInstances, you can specify a launch template +// instead of providing the launch parameters in the request. For more information, +// see Launching an instance from a launch template (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html) +// in the Amazon Elastic Compute Cloud User Guide. +// +// If you want to clone an existing launch template as the basis for creating +// a new launch template, you can use the Amazon EC2 console. The API, SDKs, +// and CLI do not support cloning a template. For more information, see Create +// a launch template from an existing launch template (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-launch-templates.html#create-launch-template-from-existing-launch-template) // in the Amazon Elastic Compute Cloud User Guide. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions @@ -9196,24 +9196,7 @@ func (c *EC2) CreateVpcEndpointRequest(input *CreateVpcEndpointInput) (req *requ // create a private connection between your VPC and the service. The service // may be provided by Amazon Web Services, an Amazon Web Services Marketplace // Partner, or another Amazon Web Services account. For more information, see -// VPC Endpoints (https://docs.aws.amazon.com/vpc/latest/userguide/vpc-endpoints.html) -// in the Amazon Virtual Private Cloud User Guide. -// -// A gateway endpoint serves as a target for a route in your route table for -// traffic destined for the Amazon Web Service. You can specify an endpoint -// policy to attach to the endpoint, which will control access to the service -// from your VPC. You can also specify the VPC route tables that use the endpoint. -// -// An interface endpoint is a network interface in your subnet that serves as -// an endpoint for communicating with the specified service. You can specify -// the subnets in which to create an endpoint, and the security groups to associate -// with the endpoint network interface. -// -// A GatewayLoadBalancer endpoint is a network interface in your subnet that -// serves an endpoint for communicating with a Gateway Load Balancer that you've -// configured as a VPC endpoint service. -// -// Use DescribeVpcEndpointServices to get a list of supported services. +// the Amazon Web Services PrivateLink Guide (https://docs.aws.amazon.com/vpc/latest/privatelink/). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -9367,26 +9350,23 @@ func (c *EC2) CreateVpcEndpointServiceConfigurationRequest(input *CreateVpcEndpo // CreateVpcEndpointServiceConfiguration API operation for Amazon Elastic Compute Cloud. // -// Creates a VPC endpoint service configuration to which service consumers (Amazon -// Web Services accounts, IAM users, and IAM roles) can connect. +// Creates a VPC endpoint service to which service consumers (Amazon Web Services +// accounts, IAM users, and IAM roles) can connect. // -// To create an endpoint service configuration, you must first create one of -// the following for your service: +// Before you create an endpoint service, you must create one of the following +// for your service: // -// * A Network Load Balancer (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html). +// * A Network Load Balancer (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/). // Service consumers connect to your service using an interface endpoint. // -// * A Gateway Load Balancer (https://docs.aws.amazon.com/elasticloadbalancing/latest/gateway/introduction.html). +// * A Gateway Load Balancer (https://docs.aws.amazon.com/elasticloadbalancing/latest/gateway/). // Service consumers connect to your service using a Gateway Load Balancer // endpoint. // -// For more information, see VPC Endpoint Services (https://docs.aws.amazon.com/vpc/latest/userguide/endpoint-service.html) -// in the Amazon Virtual Private Cloud User Guide. -// // If you set the private DNS name, you must prove that you own the private -// DNS domain name. For more information, see VPC Endpoint Service Private DNS -// Name Verification (https://docs.aws.amazon.com/vpc/latest/userguide/endpoint-services-dns-validation.html) -// in the Amazon Virtual Private Cloud User Guide. +// DNS domain name. +// +// For more information, see the Amazon Web Services PrivateLink Guide (https://docs.aws.amazon.com/vpc/latest/privatelink/). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -10662,11 +10642,6 @@ func (c *EC2) DeleteIpamRequest(input *DeleteIpamInput) (req *request.Request, o // Delete an IPAM. Deleting an IPAM removes all monitored data associated with // the IPAM including the historical data for CIDRs. // -// You cannot delete an IPAM if there are CIDRs provisioned to pools or if there -// are allocations in the pools within the IPAM. To deprovision pool CIDRs, -// see DeprovisionIpamPoolCidr (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeprovisionIpamPoolCidr.html). -// To release allocations, see ReleaseIpamPoolAllocation (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_ReleaseIpamPoolAllocation.html). -// // For more information, see Delete an IPAM (/vpc/latest/ipam/delete-ipam.html) // in the Amazon VPC IPAM User Guide. // @@ -42108,9 +42083,9 @@ func (c *EC2) ModifyVolumeRequest(input *ModifyVolumeInput) (req *request.Reques // With previous-generation instance types, resizing an EBS volume might require // detaching and reattaching the volume or stopping and restarting the instance. // -// If you reach the maximum volume modification rate per volume limit, you must -// wait at least six hours before applying further modifications to the affected -// EBS volume. +// After modifying a volume, you must wait at least six hours and ensure that +// the volume is in the in-use or available state before you can modify the +// same volume. This is sometimes referred to as a cooldown period. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -42345,8 +42320,8 @@ func (c *EC2) ModifyVpcEndpointRequest(input *ModifyVpcEndpointInput) (req *requ // // Modifies attributes of a specified VPC endpoint. The attributes that you // can modify depend on the type of VPC endpoint (interface, gateway, or Gateway -// Load Balancer). For more information, see VPC Endpoints (https://docs.aws.amazon.com/vpc/latest/userguide/vpc-endpoints.html) -// in the Amazon Virtual Private Cloud User Guide. +// Load Balancer). For more information, see the Amazon Web Services PrivateLink +// Guide (https://docs.aws.amazon.com/vpc/latest/privatelink/). // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -42502,9 +42477,7 @@ func (c *EC2) ModifyVpcEndpointServiceConfigurationRequest(input *ModifyVpcEndpo // to your endpoint service through an interface VPC endpoint. // // If you set or modify the private DNS name, you must prove that you own the -// private DNS domain name. For more information, see VPC Endpoint Service Private -// DNS Name Verification (https://docs.aws.amazon.com/vpc/latest/userguide/endpoint-services-dns-validation.html) -// in the Amazon Virtual Private Cloud User Guide. +// private DNS domain name. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -42652,9 +42625,9 @@ func (c *EC2) ModifyVpcEndpointServicePermissionsRequest(input *ModifyVpcEndpoin // ModifyVpcEndpointServicePermissions API operation for Amazon Elastic Compute Cloud. // -// Modifies the permissions for your VPC endpoint service (https://docs.aws.amazon.com/vpc/latest/userguide/endpoint-service.html). -// You can add or remove permissions for service consumers (IAM users, IAM roles, -// and Amazon Web Services accounts) to connect to your endpoint service. +// Modifies the permissions for your VPC endpoint service. You can add or remove +// permissions for service consumers (IAM users, IAM roles, and Amazon Web Services +// accounts) to connect to your endpoint service. // // If you grant permissions to all principals, the service is public. Any users // who know the name of a public service can send a request to attach an endpoint. @@ -47889,9 +47862,7 @@ func (c *EC2) StartVpcEndpointServicePrivateDnsVerificationRequest(input *StartV // consumer can use the name to access the service. // // Before the service provider runs this command, they must add a record to -// the DNS server. For more information, see Adding a TXT Record to Your Domain's -// DNS Server (https://docs.aws.amazon.com/vpc/latest/userguide/endpoint-services-dns-validation.html#add-dns-txt-record) -// in the Amazon VPC User Guide. +// the DNS server. // // Returns awserr.Error for service API and SDK errors. Use runtime type assertions // with awserr.Error's Code and Message methods to get detailed information about @@ -62383,12 +62354,17 @@ type CreateImageInput struct { // Name is a required field Name *string `locationName:"name" type:"string" required:"true"` - // By default, Amazon EC2 attempts to shut down and reboot the instance before - // creating the image. If the No Reboot option is set, Amazon EC2 doesn't shut - // down the instance before creating the image. Without a reboot, the AMI will - // be crash consistent (all the volumes are snapshotted at the same time), but - // not application consistent (all the operating system buffers are not flushed - // to disk before the snapshots are created). + // By default, when Amazon EC2 creates the new AMI, it reboots the instance + // so that it can take snapshots of the attached volumes while data is at rest, + // in order to ensure a consistent state. You can set the NoReboot parameter + // to true in the API request, or use the --no-reboot option in the CLI to prevent + // Amazon EC2 from shutting down and rebooting the instance. + // + // If you choose to bypass the shutdown and reboot process by setting the NoReboot + // parameter to true in the API request, or by using the --no-reboot option + // in the CLI, we can't guarantee the file system integrity of the created image. + // + // Default: false (follow standard reboot process) NoReboot *bool `locationName:"noReboot" type:"boolean"` // The tags to apply to the AMI and snapshots on creation. You can tag the AMI, @@ -63334,7 +63310,7 @@ type CreateKeyPairInput struct { KeyName *string `type:"string" required:"true"` // The type of key pair. Note that ED25519 keys are not supported for Windows - // instances, EC2 Instance Connect, and EC2 Serial Console. + // instances. // // Default: rsa KeyType *string `type:"string" enum:"KeyType"` @@ -69718,7 +69694,7 @@ type CreateVpcEndpointServiceConfigurationInput struct { _ struct{} `type:"structure"` // Indicates whether requests from service consumers to create an endpoint to - // your service must be accepted. To accept a request, use AcceptVpcEndpointConnections. + // your service must be accepted manually. AcceptanceRequired *bool `type:"boolean"` // Unique, case-sensitive identifier that you provide to ensure the idempotency @@ -82615,6 +82591,11 @@ type DescribeImagesInput struct { // * block-device-mapping.encrypted - A Boolean that indicates whether the // Amazon EBS volume is encrypted. // + // * creation-date - The time when the image was created, in the ISO 8601 + // format in the UTC time zone (YYYY-MM-DDThh:mm:ss.sssZ), for example, 2021-09-29T11:04:43.305Z. + // You can use a wildcard (*), for example, 2021-09-29T*, which matches an + // entire day. + // // * description - The description of the image (provided during image creation). // // * ena-support - A Boolean that indicates whether enhanced networking with @@ -86253,9 +86234,6 @@ type DescribeLocalGatewayVirtualInterfacesInput struct { // // * local-gateway-virtual-interface-id - The ID of the virtual interface. // - // * local-gateway-virtual-interface-group-id - The ID of the virtual interface - // group. - // // * owner-id - The ID of the Amazon Web Services account that owns the local // gateway virtual interface. // @@ -99700,6 +99678,9 @@ type EbsBlockDevice struct { KmsKeyId *string `type:"string"` // The ARN of the Outpost on which the snapshot is stored. + // + // This parameter is only supported on BlockDeviceMapping objects called by + // CreateImage (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateImage.html). OutpostArn *string `locationName:"outpostArn" type:"string"` // The ID of the snapshot. @@ -135959,10 +135940,7 @@ func (s *PrivateDnsDetails) SetPrivateDnsName(v string) *PrivateDnsDetails { return s } -// Information about the private DNS name for the service endpoint. For more -// information about these parameters, see VPC Endpoint Service Private DNS -// Name Verification (https://docs.aws.amazon.com/vpc/latest/userguide/ndpoint-services-dns-validation.html) -// in the Amazon Virtual Private Cloud User Guide. +// Information about the private DNS name for the service endpoint. type PrivateDnsNameConfiguration struct { _ struct{} `type:"structure"` @@ -140321,6 +140299,8 @@ func (s *RequestIpamResourceTag) SetValue(v string) *RequestIpamResourceTag { } // The information to include in the launch template. +// +// You must specify at least one parameter for the launch template data. type RequestLaunchTemplateData struct { _ struct{} `type:"structure"` @@ -145103,8 +145083,8 @@ type RunInstancesInput struct { // created, see CreateTags (https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html). TagSpecifications []*TagSpecification `locationName:"TagSpecification" locationNameList:"item" type:"list"` - // The user data to make available to the instance. For more information, see - // Run commands on your Linux instance at launch (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) + // The user data script to make available to the instance. For more information, + // see Run commands on your Linux instance at launch (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) // and Run commands on your Windows instance at launch (https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/ec2-windows-user-data.html). // If you are using a command line tool, base64-encoding is performed for you, // and you can load the text from a file. Otherwise, you must provide base64-encoded @@ -164530,6 +164510,72 @@ const ( // InstanceTypeZ1dMetal is a InstanceType enum value InstanceTypeZ1dMetal = "z1d.metal" + + // InstanceTypeX2idn16xlarge is a InstanceType enum value + InstanceTypeX2idn16xlarge = "x2idn.16xlarge" + + // InstanceTypeX2idn24xlarge is a InstanceType enum value + InstanceTypeX2idn24xlarge = "x2idn.24xlarge" + + // InstanceTypeX2idn32xlarge is a InstanceType enum value + InstanceTypeX2idn32xlarge = "x2idn.32xlarge" + + // InstanceTypeX2iednXlarge is a InstanceType enum value + InstanceTypeX2iednXlarge = "x2iedn.xlarge" + + // InstanceTypeX2iedn2xlarge is a InstanceType enum value + InstanceTypeX2iedn2xlarge = "x2iedn.2xlarge" + + // InstanceTypeX2iedn4xlarge is a InstanceType enum value + InstanceTypeX2iedn4xlarge = "x2iedn.4xlarge" + + // InstanceTypeX2iedn8xlarge is a InstanceType enum value + InstanceTypeX2iedn8xlarge = "x2iedn.8xlarge" + + // InstanceTypeX2iedn16xlarge is a InstanceType enum value + InstanceTypeX2iedn16xlarge = "x2iedn.16xlarge" + + // InstanceTypeX2iedn24xlarge is a InstanceType enum value + InstanceTypeX2iedn24xlarge = "x2iedn.24xlarge" + + // InstanceTypeX2iedn32xlarge is a InstanceType enum value + InstanceTypeX2iedn32xlarge = "x2iedn.32xlarge" + + // InstanceTypeC6aLarge is a InstanceType enum value + InstanceTypeC6aLarge = "c6a.large" + + // InstanceTypeC6aXlarge is a InstanceType enum value + InstanceTypeC6aXlarge = "c6a.xlarge" + + // InstanceTypeC6a2xlarge is a InstanceType enum value + InstanceTypeC6a2xlarge = "c6a.2xlarge" + + // InstanceTypeC6a4xlarge is a InstanceType enum value + InstanceTypeC6a4xlarge = "c6a.4xlarge" + + // InstanceTypeC6a8xlarge is a InstanceType enum value + InstanceTypeC6a8xlarge = "c6a.8xlarge" + + // InstanceTypeC6a12xlarge is a InstanceType enum value + InstanceTypeC6a12xlarge = "c6a.12xlarge" + + // InstanceTypeC6a16xlarge is a InstanceType enum value + InstanceTypeC6a16xlarge = "c6a.16xlarge" + + // InstanceTypeC6a24xlarge is a InstanceType enum value + InstanceTypeC6a24xlarge = "c6a.24xlarge" + + // InstanceTypeC6a32xlarge is a InstanceType enum value + InstanceTypeC6a32xlarge = "c6a.32xlarge" + + // InstanceTypeC6a48xlarge is a InstanceType enum value + InstanceTypeC6a48xlarge = "c6a.48xlarge" + + // InstanceTypeC6aMetal is a InstanceType enum value + InstanceTypeC6aMetal = "c6a.metal" + + // InstanceTypeM6aMetal is a InstanceType enum value + InstanceTypeM6aMetal = "m6a.metal" ) // InstanceType_Values returns all elements of the InstanceType enum @@ -165020,6 +165066,28 @@ func InstanceType_Values() []string { InstanceTypeZ1d6xlarge, InstanceTypeZ1d12xlarge, InstanceTypeZ1dMetal, + InstanceTypeX2idn16xlarge, + InstanceTypeX2idn24xlarge, + InstanceTypeX2idn32xlarge, + InstanceTypeX2iednXlarge, + InstanceTypeX2iedn2xlarge, + InstanceTypeX2iedn4xlarge, + InstanceTypeX2iedn8xlarge, + InstanceTypeX2iedn16xlarge, + InstanceTypeX2iedn24xlarge, + InstanceTypeX2iedn32xlarge, + InstanceTypeC6aLarge, + InstanceTypeC6aXlarge, + InstanceTypeC6a2xlarge, + InstanceTypeC6a4xlarge, + InstanceTypeC6a8xlarge, + InstanceTypeC6a12xlarge, + InstanceTypeC6a16xlarge, + InstanceTypeC6a24xlarge, + InstanceTypeC6a32xlarge, + InstanceTypeC6a48xlarge, + InstanceTypeC6aMetal, + InstanceTypeM6aMetal, } } diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go index 47a6b7b1d..3df30fa82 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -217,7 +217,9 @@ const ( // See VolumeExpansion for details. ControllerServiceCapability_RPC_EXPAND_VOLUME ControllerServiceCapability_RPC_Type = 9 // Indicates the SP supports the - // ListVolumesResponse.entry.published_nodes field + // ListVolumesResponse.entry.published_node_ids field and the + // ControllerGetVolumeResponse.published_node_ids field. + // The SP MUST also support PUBLISH_UNPUBLISH_VOLUME. ControllerServiceCapability_RPC_LIST_VOLUMES_PUBLISHED_NODES ControllerServiceCapability_RPC_Type = 10 // Indicates that the Controller service can report volume // conditions. @@ -2678,7 +2680,7 @@ type ControllerGetVolumeResponse_VolumeStatus struct { // A list of all the `node_id` of nodes that this volume is // controller published on. // This field is OPTIONAL. - // This field MUST be specified if the PUBLISH_UNPUBLISH_VOLUME + // This field MUST be specified if the LIST_VOLUMES_PUBLISHED_NODES // controller capability is supported. // published_node_ids MAY include nodes not published to or // reported by the SP. The CO MUST be resilient to that. @@ -5058,245 +5060,245 @@ func init() { } var fileDescriptor_9cdb00adce470e01 = []byte{ - // 3797 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6c, 0x1b, 0x49, - 0x76, 0x6a, 0xfe, 0x24, 0x3d, 0x4a, 0x32, 0x5d, 0xfa, 0x98, 0x6e, 0x49, 0x96, 0xdc, 0x1e, 0x7b, - 0x65, 0x8f, 0x4d, 0xaf, 0xb5, 0x63, 0x23, 0x23, 0x7b, 0x76, 0x87, 0xa4, 0x68, 0x89, 0x6b, 0x8a, - 0xd4, 0x34, 0x29, 0x7b, 0xed, 0x64, 0xd0, 0xd3, 0x22, 0x4b, 0x74, 0x63, 0xc8, 0x6e, 0x4e, 0x77, - 0x53, 0x91, 0xe6, 0x92, 0x20, 0x41, 0x0e, 0x41, 0x2e, 0xb9, 0xed, 0xe4, 0xb6, 0x48, 0xf6, 0x98, - 0xc5, 0x22, 0x08, 0x82, 0x1c, 0x03, 0xe4, 0x18, 0x20, 0x9b, 0xdc, 0x12, 0xe4, 0xb2, 0xb7, 0x20, - 0x58, 0x24, 0xc0, 0x5c, 0x72, 0xc9, 0x21, 0x08, 0xba, 0xaa, 0xba, 0xd9, 0x5f, 0x7e, 0x2c, 0x19, - 0x73, 0xc8, 0x49, 0xec, 0x57, 0xef, 0xbd, 0x7a, 0x55, 0xf5, 0xde, 0xab, 0xf7, 0x29, 0xc1, 0xe3, - 0xb6, 0x62, 0xbe, 0xed, 0x1f, 0xe7, 0x9a, 0x5a, 0xf7, 0x61, 0x53, 0x53, 0x4d, 0x59, 0x51, 0xb1, - 0xfe, 0xc0, 0x30, 0x35, 0x5d, 0x6e, 0xe3, 0x07, 0x8a, 0x6a, 0x62, 0xfd, 0x44, 0x6e, 0xe2, 0x87, - 0x46, 0x0f, 0x37, 0x1f, 0x36, 0x0d, 0x25, 0xd7, 0xd3, 0x35, 0x53, 0x43, 0x29, 0xeb, 0xe7, 0xe9, - 0x23, 0x7e, 0xb3, 0xad, 0x69, 0xed, 0x0e, 0x7e, 0x48, 0xa0, 0xc7, 0xfd, 0x93, 0x87, 0x2d, 0x6c, - 0x34, 0x75, 0xa5, 0x67, 0x6a, 0x3a, 0xc5, 0xe4, 0x37, 0xfc, 0x18, 0xa6, 0xd2, 0xc5, 0x86, 0x29, - 0x77, 0x7b, 0x0c, 0xe1, 0x86, 0x1f, 0xe1, 0x77, 0x75, 0xb9, 0xd7, 0xc3, 0xba, 0x41, 0xc7, 0x85, - 0x15, 0x58, 0xda, 0xc3, 0xe6, 0x61, 0xa7, 0xdf, 0x56, 0xd4, 0xb2, 0x7a, 0xa2, 0x89, 0xf8, 0xab, - 0x3e, 0x36, 0x4c, 0xe1, 0x5f, 0x39, 0x58, 0xf6, 0x0d, 0x18, 0x3d, 0x4d, 0x35, 0x30, 0x42, 0x90, - 0x50, 0xe5, 0x2e, 0xce, 0x72, 0x9b, 0xdc, 0xd6, 0xac, 0x48, 0x7e, 0xa3, 0xdb, 0xb0, 0x70, 0x8a, - 0xd5, 0x96, 0xa6, 0x4b, 0xa7, 0x58, 0x37, 0x14, 0x4d, 0xcd, 0xc6, 0xc8, 0xe8, 0x3c, 0x85, 0xbe, - 0xa4, 0x40, 0xb4, 0x07, 0x33, 0x5d, 0x59, 0x55, 0x4e, 0xb0, 0x61, 0x66, 0xe3, 0x9b, 0xf1, 0xad, - 0xf4, 0xf6, 0x87, 0x39, 0xba, 0xd4, 0x5c, 0xe8, 0x5c, 0xb9, 0x03, 0x86, 0x5d, 0x52, 0x4d, 0xfd, - 0x5c, 0x74, 0x88, 0xf9, 0xa7, 0x30, 0xef, 0x19, 0x42, 0x19, 0x88, 0x7f, 0x89, 0xcf, 0x99, 0x4c, - 0xd6, 0x4f, 0xb4, 0x04, 0xc9, 0x53, 0xb9, 0xd3, 0xc7, 0x4c, 0x12, 0xfa, 0xb1, 0x13, 0xfb, 0x2d, - 0x4e, 0xb8, 0x01, 0x6b, 0xce, 0x6c, 0x45, 0xb9, 0x27, 0x1f, 0x2b, 0x1d, 0xc5, 0x54, 0xb0, 0x61, - 0x2f, 0xfd, 0x73, 0x58, 0x8f, 0x18, 0x67, 0x3b, 0xf0, 0x0c, 0xe6, 0x9a, 0x2e, 0x78, 0x96, 0x23, - 0x4b, 0xc9, 0xda, 0x4b, 0xf1, 0x51, 0x9e, 0x8b, 0x1e, 0x6c, 0xe1, 0x57, 0x71, 0xc8, 0xf8, 0x51, - 0xd0, 0x33, 0x98, 0x36, 0xb0, 0x7e, 0xaa, 0x34, 0xe9, 0xbe, 0xa6, 0xb7, 0x37, 0xa3, 0xb8, 0xe5, - 0xea, 0x14, 0x6f, 0x7f, 0x4a, 0xb4, 0x49, 0xd0, 0x11, 0x64, 0x4e, 0xb5, 0x4e, 0xbf, 0x8b, 0x25, - 0x7c, 0xd6, 0x93, 0x55, 0xe7, 0x00, 0xd2, 0xdb, 0x5b, 0x91, 0x6c, 0x5e, 0x12, 0x82, 0x92, 0x8d, - 0xbf, 0x3f, 0x25, 0x5e, 0x39, 0xf5, 0x82, 0xf8, 0x9f, 0x72, 0x30, 0xcd, 0x66, 0x43, 0x1f, 0x43, - 0xc2, 0x3c, 0xef, 0x51, 0xe9, 0x16, 0xb6, 0x6f, 0x8f, 0x92, 0x2e, 0xd7, 0x38, 0xef, 0x61, 0x91, - 0x90, 0x08, 0x9f, 0x41, 0xc2, 0xfa, 0x42, 0x69, 0x98, 0x3e, 0xaa, 0xbe, 0xa8, 0xd6, 0x5e, 0x55, - 0x33, 0x53, 0x68, 0x05, 0x50, 0xb1, 0x56, 0x6d, 0x88, 0xb5, 0x4a, 0xa5, 0x24, 0x4a, 0xf5, 0x92, - 0xf8, 0xb2, 0x5c, 0x2c, 0x65, 0x38, 0xf4, 0x01, 0x6c, 0xbe, 0xac, 0x55, 0x8e, 0x0e, 0x4a, 0x52, - 0xbe, 0x58, 0x2c, 0xd5, 0xeb, 0xe5, 0x42, 0xb9, 0x52, 0x6e, 0xbc, 0x96, 0x8a, 0xb5, 0x6a, 0xbd, - 0x21, 0xe6, 0xcb, 0xd5, 0x46, 0x3d, 0x13, 0xe3, 0xff, 0x80, 0x83, 0x2b, 0xbe, 0x05, 0xa0, 0xbc, - 0x47, 0xc2, 0x07, 0xe3, 0x2e, 0xdc, 0x2d, 0xe9, 0xfd, 0x30, 0x49, 0x01, 0x52, 0xb5, 0x6a, 0xa5, - 0x5c, 0xb5, 0xa4, 0x4b, 0xc3, 0x74, 0xed, 0xf9, 0x73, 0xf2, 0x11, 0x2b, 0xa4, 0xe8, 0x84, 0xc2, - 0x02, 0xcc, 0x1d, 0xea, 0xda, 0x31, 0xb6, 0xf5, 0x27, 0x0f, 0xf3, 0xec, 0x9b, 0xe9, 0xcb, 0xf7, - 0x21, 0xa9, 0x63, 0xb9, 0x75, 0xce, 0x8e, 0x96, 0xcf, 0x51, 0x9b, 0xcc, 0xd9, 0x36, 0x99, 0x2b, - 0x68, 0x5a, 0xe7, 0xa5, 0xa5, 0x9f, 0x22, 0x45, 0x14, 0xbe, 0x4d, 0xc0, 0x62, 0x51, 0xc7, 0xb2, - 0x89, 0xa9, 0xb4, 0x8c, 0x75, 0xa8, 0xed, 0x3d, 0x83, 0x05, 0x4b, 0xbf, 0x9a, 0x8a, 0x79, 0x2e, - 0xe9, 0xb2, 0xda, 0xc6, 0xec, 0xe8, 0x97, 0xed, 0x1d, 0x28, 0xb2, 0x51, 0xd1, 0x1a, 0x14, 0xe7, - 0x9b, 0xee, 0x4f, 0x54, 0x86, 0x45, 0xa6, 0x3a, 0x1e, 0x95, 0x8e, 0x7b, 0x55, 0x9a, 0x4a, 0xe1, - 0x52, 0x69, 0x74, 0xea, 0x85, 0x28, 0xd8, 0x40, 0x2f, 0x00, 0x7a, 0xb2, 0x2e, 0x77, 0xb1, 0x89, - 0x75, 0x23, 0x9b, 0xf0, 0xda, 0x77, 0xc8, 0x6a, 0x72, 0x87, 0x0e, 0x36, 0xb5, 0x6f, 0x17, 0x39, - 0xda, 0xb3, 0x0c, 0xa2, 0xa9, 0x63, 0xd3, 0xc8, 0x26, 0x09, 0xa7, 0xad, 0x61, 0x9c, 0xea, 0x14, - 0x95, 0xb0, 0x29, 0xc4, 0xbf, 0x29, 0x70, 0xa2, 0x4d, 0x8d, 0x6a, 0xb0, 0x6c, 0x2f, 0x50, 0x53, - 0x4d, 0xac, 0x9a, 0x92, 0xa1, 0xf5, 0xf5, 0x26, 0xce, 0xa6, 0xc8, 0x2e, 0xad, 0xfa, 0x96, 0x48, - 0x71, 0xea, 0x04, 0x45, 0x64, 0x5b, 0xe3, 0x01, 0xa2, 0x37, 0xc0, 0xcb, 0xcd, 0x26, 0x36, 0x0c, - 0x85, 0xee, 0x85, 0xa4, 0xe3, 0xaf, 0xfa, 0x8a, 0x8e, 0xbb, 0x58, 0x35, 0x8d, 0xec, 0xb4, 0x97, - 0x6b, 0x43, 0xeb, 0x69, 0x1d, 0xad, 0x7d, 0x2e, 0x0e, 0x70, 0xc4, 0xeb, 0x1e, 0x72, 0xd7, 0x88, - 0xc1, 0x7f, 0x02, 0x57, 0x7c, 0x9b, 0x32, 0x89, 0x67, 0xe3, 0x77, 0x60, 0xce, 0xbd, 0x13, 0x13, - 0x79, 0xc5, 0x3f, 0x89, 0xc1, 0x62, 0xc8, 0x1e, 0xa0, 0x7d, 0x98, 0x31, 0x54, 0xb9, 0x67, 0xbc, - 0xd5, 0x4c, 0xa6, 0xbf, 0xf7, 0x86, 0x6c, 0x59, 0xae, 0xce, 0x70, 0xe9, 0xe7, 0xfe, 0x94, 0xe8, - 0x50, 0xa3, 0x02, 0xa4, 0xe8, 0x7e, 0xfa, 0x7d, 0x53, 0x18, 0x1f, 0x0a, 0x73, 0xb8, 0x30, 0x4a, - 0xfe, 0x11, 0x2c, 0x78, 0x67, 0x40, 0x1b, 0x90, 0xb6, 0x67, 0x90, 0x94, 0x16, 0x5b, 0x2b, 0xd8, - 0xa0, 0x72, 0x8b, 0xff, 0x10, 0xe6, 0xdc, 0xcc, 0xd0, 0x2a, 0xcc, 0x32, 0x85, 0x70, 0xd0, 0x67, - 0x28, 0xa0, 0xdc, 0x72, 0x6c, 0xfa, 0x87, 0xb0, 0xe4, 0xd5, 0x33, 0x66, 0xca, 0x77, 0x9c, 0x35, - 0xd0, 0xbd, 0x58, 0xf0, 0xae, 0xc1, 0x96, 0x53, 0xf8, 0x79, 0x12, 0x32, 0x7e, 0xa3, 0x41, 0xcf, - 0x20, 0x79, 0xdc, 0xd1, 0x9a, 0x5f, 0x32, 0xda, 0x0f, 0xa2, 0xac, 0x2b, 0x57, 0xb0, 0xb0, 0x28, - 0x74, 0x7f, 0x4a, 0xa4, 0x44, 0x16, 0x75, 0x57, 0xeb, 0xab, 0x26, 0xdb, 0xbd, 0x68, 0xea, 0x03, - 0x0b, 0x6b, 0x40, 0x4d, 0x88, 0xd0, 0x2e, 0xa4, 0xa9, 0xda, 0x49, 0x5d, 0xad, 0x85, 0xb3, 0x71, - 0xc2, 0xe3, 0x56, 0x24, 0x8f, 0x3c, 0xc1, 0x3d, 0xd0, 0x5a, 0x58, 0x04, 0xd9, 0xf9, 0xcd, 0xcf, - 0x43, 0xda, 0x25, 0x1b, 0xff, 0x35, 0xa4, 0x5d, 0x93, 0xa1, 0x6b, 0x30, 0x7d, 0x62, 0x48, 0x8e, - 0x13, 0x9e, 0x15, 0x53, 0x27, 0x06, 0xf1, 0xa7, 0x1b, 0x90, 0x26, 0x52, 0x48, 0x27, 0x1d, 0xb9, - 0x6d, 0x64, 0x63, 0x9b, 0x71, 0xeb, 0x8c, 0x08, 0xe8, 0xb9, 0x05, 0x41, 0x8f, 0x80, 0x39, 0x14, - 0x89, 0xe2, 0xb5, 0x75, 0xad, 0xdf, 0x23, 0x42, 0xce, 0x16, 0xe2, 0x3f, 0x2b, 0x70, 0x22, 0xbb, - 0xdf, 0xc8, 0x6c, 0x7b, 0xd6, 0x20, 0xff, 0xd7, 0x31, 0x80, 0x81, 0x94, 0xe8, 0x19, 0x24, 0xc8, - 0xc2, 0xa8, 0xf7, 0xdf, 0x1a, 0x63, 0x61, 0x39, 0xb2, 0x3a, 0x42, 0x25, 0xfc, 0x3b, 0x07, 0x09, - 0xc2, 0xc6, 0x7f, 0x47, 0xd5, 0xcb, 0xd5, 0xbd, 0x4a, 0x49, 0xaa, 0xd6, 0x76, 0x4b, 0xd2, 0x2b, - 0xb1, 0xdc, 0x28, 0x89, 0x19, 0x0e, 0xad, 0xc2, 0x35, 0x37, 0x5c, 0x2c, 0xe5, 0x77, 0x4b, 0xa2, - 0x54, 0xab, 0x56, 0x5e, 0x67, 0x62, 0x88, 0x87, 0x95, 0x83, 0xa3, 0x4a, 0xa3, 0x1c, 0x1c, 0x8b, - 0xa3, 0x35, 0xc8, 0xba, 0xc6, 0x18, 0x0f, 0xc6, 0x36, 0x61, 0xb1, 0x75, 0x8d, 0xd2, 0x9f, 0x6c, - 0x30, 0x89, 0x04, 0xb8, 0xee, 0x9e, 0xd3, 0x4b, 0x9b, 0xe2, 0xad, 0x4d, 0x42, 0x37, 0x21, 0xeb, - 0xc6, 0xf1, 0x70, 0x98, 0x26, 0x28, 0x85, 0x79, 0x47, 0x0d, 0x88, 0x9a, 0xbf, 0x82, 0x79, 0xcf, - 0xed, 0x60, 0x05, 0x72, 0xcc, 0x9d, 0xb5, 0xa4, 0xe3, 0x73, 0x93, 0x04, 0x37, 0xdc, 0x56, 0x5c, - 0x9c, 0xb7, 0xa1, 0x05, 0x0b, 0x68, 0x1d, 0x68, 0x47, 0xe9, 0x2a, 0x26, 0xc3, 0x89, 0x11, 0x1c, - 0x20, 0x20, 0x82, 0x20, 0xfc, 0x3a, 0x06, 0x29, 0xa6, 0x15, 0xb7, 0x5d, 0xf7, 0x93, 0x87, 0xa5, - 0x0d, 0xa5, 0x2c, 0x3d, 0x66, 0x19, 0xf3, 0x9a, 0x25, 0xda, 0x87, 0x05, 0xb7, 0x13, 0x3f, 0xb3, - 0xc3, 0xc7, 0x9b, 0xde, 0x73, 0x76, 0x7b, 0x92, 0x33, 0x16, 0x34, 0xce, 0x9f, 0xba, 0x61, 0xa8, - 0x00, 0x0b, 0xbe, 0x7b, 0x20, 0x31, 0xfa, 0x1e, 0x98, 0x6f, 0x7a, 0x5c, 0x62, 0x1e, 0x16, 0x6d, - 0x17, 0xde, 0xc1, 0x92, 0xc9, 0x5c, 0x3c, 0xbb, 0xa7, 0x32, 0x01, 0xd7, 0x8f, 0x06, 0xc8, 0x36, - 0x8c, 0xff, 0x14, 0x50, 0x50, 0xd6, 0x89, 0xfc, 0x75, 0x1f, 0x16, 0x43, 0x2e, 0x17, 0x94, 0x83, - 0x59, 0x72, 0x54, 0x86, 0x62, 0x62, 0x16, 0x98, 0x06, 0x25, 0x1a, 0xa0, 0x58, 0xf8, 0x3d, 0x1d, - 0x9f, 0x60, 0x5d, 0xc7, 0x2d, 0x62, 0x98, 0xa1, 0xf8, 0x0e, 0x8a, 0xf0, 0x87, 0x1c, 0xcc, 0xd8, - 0x70, 0xb4, 0x03, 0x33, 0x06, 0x6e, 0xd3, 0x8b, 0x8f, 0xce, 0x75, 0xc3, 0x4f, 0x9b, 0xab, 0x33, - 0x04, 0x16, 0xc2, 0xdb, 0xf8, 0x56, 0x08, 0xef, 0x19, 0x9a, 0x68, 0xf1, 0x7f, 0xcb, 0xc1, 0xe2, - 0x2e, 0xee, 0x60, 0x7f, 0x7c, 0x34, 0xcc, 0xb7, 0xbb, 0x43, 0x8a, 0x98, 0x37, 0xa4, 0x08, 0x61, - 0x35, 0x24, 0xa4, 0xb8, 0xd0, 0x35, 0xbb, 0x02, 0x4b, 0xde, 0xd9, 0xe8, 0xc5, 0x22, 0xfc, 0x57, - 0x1c, 0x6e, 0x58, 0xba, 0xa0, 0x6b, 0x9d, 0x0e, 0xd6, 0x0f, 0xfb, 0xc7, 0x1d, 0xc5, 0x78, 0x3b, - 0xc1, 0xe2, 0xae, 0xc1, 0xb4, 0xaa, 0xb5, 0x5c, 0xc6, 0x93, 0xb2, 0x3e, 0xcb, 0x2d, 0x54, 0x82, - 0xab, 0xfe, 0x00, 0xef, 0x9c, 0xb9, 0xff, 0xe8, 0xf0, 0x2e, 0x73, 0xea, 0xbf, 0xbb, 0x78, 0x98, - 0xb1, 0x42, 0x53, 0x4d, 0xed, 0x9c, 0x13, 0x8b, 0x99, 0x11, 0x9d, 0x6f, 0x24, 0xfa, 0x63, 0xb5, - 0x1f, 0x38, 0xb1, 0xda, 0xd0, 0x15, 0x0d, 0x0b, 0xdb, 0xbe, 0x08, 0x58, 0x7c, 0x8a, 0xb0, 0xfe, - 0x78, 0x4c, 0xd6, 0x23, 0x3d, 0xc1, 0x45, 0x4e, 0xf1, 0x12, 0xcc, 0xf7, 0x1f, 0x38, 0xd8, 0x88, - 0x5c, 0x02, 0x0b, 0x36, 0x5a, 0x70, 0xa5, 0x47, 0x07, 0x9c, 0x4d, 0xa0, 0x56, 0xf6, 0x74, 0xe4, - 0x26, 0xb0, 0xfc, 0x99, 0x41, 0x3d, 0xdb, 0xb0, 0xd0, 0xf3, 0x00, 0xf9, 0x3c, 0x2c, 0x86, 0xa0, - 0x4d, 0xb4, 0x98, 0xdf, 0x70, 0xb0, 0x39, 0x10, 0xe5, 0x48, 0xed, 0x5d, 0x9e, 0xfa, 0x36, 0x06, - 0xba, 0x45, 0x5d, 0xfe, 0xe3, 0xe0, 0xda, 0xc3, 0x27, 0x7c, 0x5f, 0x16, 0x7c, 0x0b, 0x6e, 0x0e, - 0x99, 0x9a, 0x99, 0xf3, 0xaf, 0x13, 0x70, 0xf3, 0xa5, 0xdc, 0x51, 0x5a, 0x4e, 0x08, 0x19, 0x52, - 0x69, 0x18, 0xbe, 0x25, 0xcd, 0x80, 0x05, 0x50, 0xaf, 0xf5, 0xcc, 0xb1, 0xda, 0x51, 0xfc, 0xc7, - 0xb8, 0x0e, 0x2f, 0x31, 0xfd, 0x7b, 0x1d, 0x92, 0xfe, 0x7d, 0x3c, 0xbe, 0xac, 0xc3, 0x92, 0xc1, - 0x23, 0xbf, 0x83, 0x79, 0x32, 0x3e, 0xdf, 0x21, 0x5a, 0x70, 0x61, 0x2b, 0xfe, 0x2e, 0xf3, 0xb5, - 0xbf, 0x4f, 0x80, 0x30, 0x6c, 0xf5, 0xcc, 0x87, 0x88, 0x30, 0xdb, 0xd4, 0xd4, 0x13, 0x45, 0xef, - 0xe2, 0x16, 0xcb, 0x3b, 0x3e, 0x1a, 0x67, 0xf3, 0x98, 0x03, 0x29, 0xda, 0xb4, 0xe2, 0x80, 0x0d, - 0xca, 0xc2, 0x74, 0x17, 0x1b, 0x86, 0xdc, 0xb6, 0xc5, 0xb2, 0x3f, 0xf9, 0x5f, 0xc4, 0x61, 0xd6, - 0x21, 0x41, 0x6a, 0x40, 0x83, 0xa9, 0xfb, 0xda, 0x7b, 0x17, 0x01, 0xde, 0x5d, 0x99, 0x63, 0xef, - 0xa0, 0xcc, 0x2d, 0x8f, 0x32, 0x53, 0x73, 0xd8, 0x7d, 0x27, 0xb1, 0x87, 0xe8, 0xf5, 0x77, 0xae, - 0x80, 0xc2, 0xef, 0x00, 0xaa, 0x28, 0x06, 0xcb, 0xdf, 0x1c, 0xb7, 0x64, 0xa5, 0x6b, 0xf2, 0x99, - 0x84, 0x55, 0x53, 0x57, 0x58, 0xb8, 0x9e, 0x14, 0xa1, 0x2b, 0x9f, 0x95, 0x28, 0xc4, 0x0a, 0xe9, - 0x0d, 0x53, 0xd6, 0x4d, 0x45, 0x6d, 0x4b, 0xa6, 0xf6, 0x25, 0x76, 0xca, 0xbd, 0x36, 0xb4, 0x61, - 0x01, 0x85, 0xff, 0x8c, 0xc1, 0xa2, 0x87, 0x3d, 0xd3, 0xc9, 0xa7, 0x30, 0x3d, 0xe0, 0xed, 0x09, - 0xe3, 0x43, 0xb0, 0x73, 0x74, 0xdb, 0x6c, 0x0a, 0xb4, 0x0e, 0xa0, 0xe2, 0x33, 0xd3, 0x33, 0xef, - 0xac, 0x05, 0x21, 0x73, 0xf2, 0x7f, 0xc4, 0x39, 0xe9, 0xbe, 0x29, 0x9b, 0x7d, 0x03, 0xdd, 0x07, - 0xc4, 0x5c, 0x34, 0x6e, 0x49, 0xec, 0x8e, 0xa1, 0xf3, 0xce, 0x8a, 0x19, 0x67, 0xa4, 0x4a, 0x6e, - 0x1b, 0x03, 0xed, 0x39, 0x95, 0xd4, 0xa6, 0xa6, 0xb6, 0x14, 0x73, 0x50, 0x49, 0xbd, 0x16, 0x48, - 0x10, 0xe8, 0x30, 0xcd, 0x4f, 0xaf, 0x9c, 0x7a, 0xa1, 0xfc, 0x57, 0x90, 0xa4, 0xc7, 0x31, 0x66, - 0xc5, 0x00, 0x7d, 0x0a, 0x29, 0x83, 0x48, 0xec, 0xaf, 0x8e, 0x84, 0xed, 0x89, 0x7b, 0x85, 0x22, - 0xa3, 0x13, 0x7e, 0x08, 0xfc, 0xe0, 0x62, 0xda, 0xc3, 0xe6, 0xf8, 0xd7, 0xef, 0x8e, 0xb5, 0x06, - 0xe1, 0xa7, 0x31, 0x58, 0x0d, 0x65, 0x30, 0x59, 0xed, 0x03, 0xed, 0xfb, 0x56, 0xf2, 0xfd, 0xe0, - 0x8d, 0x1d, 0x60, 0x1e, 0xba, 0x22, 0xfe, 0xf7, 0x2f, 0x76, 0x98, 0x85, 0x89, 0x0f, 0x33, 0x70, - 0x8e, 0x74, 0x67, 0x7e, 0x11, 0x03, 0xb4, 0x87, 0x4d, 0x27, 0x55, 0x66, 0x5b, 0x1a, 0xe1, 0x6f, - 0xb8, 0x77, 0xf0, 0x37, 0x3f, 0xf6, 0xf8, 0x1b, 0xea, 0xb1, 0xee, 0xb9, 0x7a, 0x23, 0xbe, 0xa9, - 0x87, 0xde, 0x96, 0x11, 0xe9, 0x29, 0x8d, 0xf9, 0xc7, 0x4b, 0x4f, 0x2f, 0xe8, 0x56, 0xfe, 0x83, - 0x83, 0x45, 0x8f, 0xd0, 0x4c, 0x83, 0x1e, 0x00, 0x92, 0x4f, 0x65, 0xa5, 0x23, 0x5b, 0x82, 0xd9, - 0xe9, 0x3f, 0x2b, 0x07, 0x5c, 0x75, 0x46, 0x6c, 0x32, 0x74, 0x08, 0x8b, 0x5d, 0xf9, 0x4c, 0xe9, - 0xf6, 0xbb, 0x12, 0xdb, 0x67, 0x43, 0xf9, 0xda, 0xae, 0x1e, 0xae, 0x06, 0xaa, 0xe8, 0x65, 0xd5, - 0x7c, 0xf2, 0x11, 0x29, 0xa3, 0x53, 0x9b, 0xbc, 0xca, 0x88, 0x99, 0x06, 0x29, 0x5f, 0x63, 0xc2, - 0x51, 0x51, 0x03, 0x1c, 0xe3, 0x63, 0x73, 0xa4, 0xc4, 0x03, 0x8e, 0x82, 0xe0, 0x8e, 0x7c, 0xd9, - 0x9a, 0xfd, 0x0d, 0xa5, 0x8e, 0x3b, 0x62, 0x0c, 0xe0, 0xb0, 0xbd, 0xd9, 0x0b, 0x6d, 0x2a, 0xdd, - 0x0a, 0xda, 0x0e, 0xeb, 0xb0, 0x44, 0xf6, 0x97, 0xfe, 0x37, 0xee, 0x36, 0xe3, 0x00, 0x36, 0x7a, - 0x0a, 0x71, 0xbd, 0xd7, 0x64, 0x36, 0xfc, 0xbd, 0x31, 0xf8, 0xe7, 0xc4, 0xc3, 0xe2, 0xfe, 0x94, - 0x68, 0x51, 0xf1, 0x7f, 0x16, 0x87, 0xb8, 0x78, 0x58, 0x44, 0x9f, 0x7a, 0x9a, 0x2d, 0xf7, 0xc7, - 0xe4, 0xe2, 0xee, 0xb5, 0xfc, 0x53, 0x2c, 0xac, 0xd9, 0x92, 0x85, 0xa5, 0xa2, 0x58, 0xca, 0x37, - 0x4a, 0xd2, 0x6e, 0xa9, 0x52, 0x6a, 0x94, 0x24, 0xda, 0x0c, 0xca, 0x70, 0x68, 0x0d, 0xb2, 0x87, - 0x47, 0x85, 0x4a, 0xb9, 0xbe, 0x2f, 0x1d, 0x55, 0xed, 0x5f, 0x6c, 0x34, 0x86, 0x32, 0x30, 0x57, - 0x29, 0xd7, 0x1b, 0x0c, 0x50, 0xcf, 0xc4, 0x2d, 0xc8, 0x5e, 0xa9, 0x21, 0x15, 0xf3, 0x87, 0xf9, - 0x62, 0xb9, 0xf1, 0x3a, 0x93, 0x40, 0x3c, 0xac, 0x78, 0x79, 0xd7, 0xab, 0xf9, 0xc3, 0xfa, 0x7e, - 0xad, 0x91, 0x49, 0x22, 0x04, 0x0b, 0x84, 0xde, 0x06, 0xd5, 0x33, 0x29, 0x8b, 0x43, 0xb1, 0x52, - 0xab, 0x3a, 0x32, 0x4c, 0xa3, 0x25, 0xc8, 0xd8, 0x33, 0x8b, 0xa5, 0xfc, 0x2e, 0xa9, 0xea, 0xcd, - 0xa0, 0xab, 0x30, 0x5f, 0xfa, 0xc9, 0x61, 0xbe, 0xba, 0x6b, 0x23, 0xce, 0xa2, 0x4d, 0x58, 0x73, - 0x8b, 0x23, 0x31, 0xaa, 0xd2, 0x2e, 0xa9, 0xcc, 0xd5, 0x33, 0x80, 0xae, 0x43, 0x86, 0xf5, 0xb9, - 0x8a, 0xb5, 0xea, 0x6e, 0xb9, 0x51, 0xae, 0x55, 0x33, 0x69, 0x5a, 0xc6, 0x5b, 0x04, 0xb0, 0x24, - 0x67, 0xcc, 0xe6, 0x46, 0xd7, 0xf6, 0xe6, 0x69, 0x6d, 0xcf, 0xae, 0x5d, 0xff, 0x26, 0x06, 0xcb, - 0xb4, 0x78, 0x6d, 0x97, 0xca, 0x6d, 0x87, 0xb5, 0x05, 0x19, 0x5a, 0xf4, 0x92, 0xfc, 0x57, 0xc1, - 0x02, 0x85, 0xbf, 0xb4, 0x93, 0x0f, 0xbb, 0xd1, 0x14, 0x73, 0x35, 0x9a, 0xca, 0xfe, 0x54, 0xec, - 0x9e, 0xb7, 0x25, 0xe3, 0x9b, 0x6d, 0x58, 0x76, 0x7f, 0x10, 0x92, 0x2b, 0x3c, 0x18, 0xce, 0x6d, - 0x58, 0x1c, 0x75, 0x91, 0x54, 0xfe, 0x82, 0xae, 0xee, 0x39, 0xac, 0xf8, 0xe5, 0x65, 0x06, 0x7d, - 0x3f, 0xd0, 0x38, 0x71, 0x7c, 0xaf, 0x83, 0xeb, 0x60, 0x08, 0xff, 0xc2, 0xc1, 0x8c, 0x0d, 0xb6, - 0x62, 0x1c, 0xcb, 0x2f, 0x79, 0xca, 0xa5, 0xb3, 0x16, 0xc4, 0xa9, 0xbe, 0xba, 0x5b, 0x1e, 0x31, - 0x7f, 0xcb, 0x23, 0xf4, 0x9c, 0xe3, 0xa1, 0xe7, 0xfc, 0x23, 0x98, 0x6f, 0x5a, 0xe2, 0x2b, 0x9a, - 0x2a, 0x99, 0x4a, 0xd7, 0xae, 0x86, 0x06, 0x5b, 0x94, 0x0d, 0xfb, 0x5d, 0x81, 0x38, 0x67, 0x13, - 0x58, 0x20, 0xb4, 0x09, 0x73, 0xa4, 0x65, 0x29, 0x99, 0x9a, 0xd4, 0x37, 0x70, 0x36, 0x49, 0x6a, - 0x43, 0x40, 0x60, 0x0d, 0xed, 0xc8, 0xc0, 0xc2, 0xdf, 0x71, 0xb0, 0x4c, 0x4b, 0x5e, 0x7e, 0x75, - 0x1c, 0xd5, 0xba, 0x71, 0x6b, 0x9c, 0xef, 0x4a, 0x0c, 0x65, 0xf8, 0xbe, 0x32, 0xfe, 0x2c, 0xac, - 0xf8, 0xe7, 0x63, 0x69, 0xfe, 0x2f, 0x63, 0xb0, 0x64, 0xc5, 0x67, 0xf6, 0xc0, 0x65, 0x87, 0xd0, - 0x13, 0x9c, 0xa4, 0x6f, 0x33, 0x13, 0x81, 0xcd, 0xdc, 0xf7, 0x27, 0xd1, 0x77, 0xdd, 0x11, 0xa6, - 0x7f, 0x05, 0xef, 0x6b, 0x2f, 0xff, 0x92, 0x83, 0x65, 0xdf, 0x7c, 0xcc, 0x5e, 0x3e, 0xf1, 0x67, - 0x05, 0xb7, 0x22, 0xe4, 0x7b, 0xa7, 0xbc, 0xe0, 0xb1, 0x1d, 0x8f, 0x4f, 0x66, 0x96, 0xff, 0x1c, - 0x83, 0xf5, 0xc1, 0xa5, 0x46, 0x1e, 0x0d, 0xb4, 0x26, 0x28, 0x6b, 0x5d, 0xac, 0x37, 0xff, 0x99, - 0xdf, 0xe1, 0x6e, 0x07, 0xef, 0xd9, 0x10, 0x91, 0x86, 0x39, 0xde, 0xd0, 0x6a, 0x70, 0x62, 0xd2, - 0x6a, 0xf0, 0x85, 0x34, 0xe0, 0xf7, 0xdc, 0x85, 0x6e, 0xaf, 0xf8, 0x4c, 0x13, 0xc6, 0xec, 0x18, - 0x3d, 0x81, 0x6b, 0x24, 0x05, 0x70, 0xde, 0xbc, 0xd8, 0x9d, 0x78, 0xea, 0x12, 0x67, 0xc4, 0x65, - 0x6b, 0xd8, 0x79, 0xe8, 0xc1, 0xba, 0x24, 0x2d, 0xe1, 0xdb, 0x04, 0xac, 0x58, 0x29, 0x42, 0xdd, - 0x94, 0xdb, 0x93, 0xf4, 0x0f, 0x7e, 0x3b, 0x58, 0x8e, 0x8d, 0x79, 0x8f, 0x25, 0x9c, 0xeb, 0x38, - 0x55, 0x58, 0x94, 0x83, 0x45, 0xc3, 0x94, 0xdb, 0xc4, 0x1d, 0xc8, 0x7a, 0x1b, 0x9b, 0x52, 0x4f, - 0x36, 0xdf, 0x32, 0x5b, 0xbf, 0xca, 0x86, 0x1a, 0x64, 0xe4, 0x50, 0x36, 0xdf, 0x5e, 0xd2, 0x41, - 0xa2, 0x1f, 0xfb, 0x9d, 0xc2, 0x87, 0x23, 0xd6, 0x32, 0x44, 0xb7, 0x7e, 0x12, 0x51, 0xb2, 0x7f, - 0x34, 0x82, 0xe5, 0xe8, 0x52, 0xfd, 0xc5, 0x4b, 0xd4, 0xdf, 0x71, 0xb5, 0xff, 0x3a, 0x5c, 0x0b, - 0x2c, 0x9e, 0x5d, 0x21, 0x6d, 0xc8, 0x5a, 0x43, 0x47, 0xaa, 0x31, 0xa1, 0x3a, 0x46, 0x68, 0x4c, - 0x2c, 0x42, 0x63, 0x84, 0x55, 0xb8, 0x1e, 0x32, 0x11, 0x93, 0xe2, 0x6f, 0x92, 0x54, 0x8c, 0xc9, - 0x1b, 0x4f, 0x9f, 0x47, 0x59, 0xc5, 0x47, 0xee, 0x63, 0x0f, 0xed, 0xd1, 0xbc, 0x0f, 0xbb, 0xd8, - 0x80, 0xb4, 0x1b, 0x8f, 0x5d, 0x83, 0xe6, 0x08, 0xc3, 0x49, 0x5e, 0xa8, 0x1f, 0x96, 0xf2, 0xf5, - 0xc3, 0x2a, 0x03, 0xa3, 0x9a, 0xf6, 0x86, 0xb6, 0x91, 0x5b, 0x31, 0xc4, 0xac, 0xde, 0x04, 0xcc, - 0x6a, 0xc6, 0xdb, 0x64, 0x8b, 0x64, 0xfa, 0xff, 0xc0, 0xb0, 0x98, 0x52, 0x87, 0x76, 0xbf, 0x84, - 0x37, 0xc0, 0x53, 0x8d, 0x9f, 0xbc, 0x1f, 0xe5, 0x53, 0xa3, 0x98, 0x5f, 0x8d, 0x84, 0x75, 0x58, - 0x0d, 0xe5, 0xcd, 0xa6, 0xfe, 0x63, 0x8e, 0x0a, 0xe6, 0x14, 0xba, 0xea, 0xa6, 0x6c, 0x1a, 0xe3, - 0x4e, 0xcd, 0x06, 0xdd, 0x53, 0x53, 0x10, 0xd1, 0xe0, 0x09, 0x4d, 0x42, 0xf8, 0x53, 0x8e, 0xee, - 0x83, 0x5f, 0x16, 0x76, 0xdb, 0xde, 0x85, 0x64, 0x9f, 0xd4, 0xf2, 0x69, 0xd4, 0xb5, 0xe8, 0x35, - 0x82, 0x23, 0x6b, 0x48, 0xa4, 0x18, 0x97, 0x56, 0x1d, 0x15, 0x7e, 0xc9, 0x41, 0xda, 0xc5, 0x1f, - 0xad, 0xc1, 0xac, 0x53, 0xfe, 0xb1, 0xf3, 0x1d, 0x07, 0x60, 0x1d, 0xbf, 0xa9, 0x99, 0x72, 0x87, - 0xbd, 0x33, 0xa1, 0x1f, 0x56, 0x8a, 0xda, 0x37, 0x30, 0x0d, 0x87, 0xe3, 0x22, 0xf9, 0x8d, 0xee, - 0x43, 0xa2, 0xaf, 0x2a, 0x26, 0x31, 0xfb, 0x05, 0xbf, 0x3d, 0x93, 0xa9, 0x72, 0x47, 0xaa, 0x62, - 0x8a, 0x04, 0x4b, 0xb8, 0x07, 0x09, 0xeb, 0xcb, 0x5b, 0x81, 0x98, 0x85, 0x64, 0xe1, 0x75, 0xa3, - 0x54, 0xcf, 0x70, 0x08, 0x20, 0x55, 0xa6, 0xf9, 0x7a, 0x4c, 0xa8, 0xd8, 0x0f, 0x4e, 0x9d, 0x45, - 0x58, 0x2e, 0x40, 0x3e, 0x56, 0x35, 0xbd, 0x2b, 0x77, 0x88, 0xcc, 0x33, 0xa2, 0xf3, 0x1d, 0xdd, - 0x22, 0xa1, 0x05, 0xc5, 0x35, 0xe7, 0x44, 0xc2, 0xea, 0x45, 0x5f, 0x50, 0xdd, 0x8a, 0xaa, 0x14, - 0xe5, 0x43, 0x2b, 0x45, 0xeb, 0x9e, 0x5b, 0x76, 0x44, 0x8d, 0xe8, 0x57, 0x31, 0x58, 0x0e, 0xc5, - 0x43, 0x8f, 0xdd, 0xd5, 0xa1, 0x9b, 0x43, 0x79, 0xba, 0xeb, 0x42, 0xff, 0xcd, 0xd1, 0xba, 0xd0, - 0x8e, 0xa7, 0x2e, 0x74, 0x67, 0x24, 0xbd, 0xbb, 0x22, 0xf4, 0x57, 0x5c, 0x44, 0x45, 0xa8, 0xde, - 0xc8, 0xef, 0x95, 0xa4, 0xa3, 0x2a, 0xfd, 0xeb, 0x54, 0x84, 0x96, 0x20, 0x33, 0xa8, 0x93, 0x48, - 0xf5, 0x46, 0xbe, 0x51, 0xcf, 0xc4, 0x82, 0xd5, 0x98, 0x78, 0x68, 0xad, 0x25, 0x31, 0xba, 0xac, - 0x92, 0xa4, 0x28, 0xab, 0x80, 0x18, 0xf5, 0x41, 0xed, 0xa8, 0xda, 0x90, 0xf6, 0xc4, 0xda, 0xd1, - 0x21, 0x7b, 0x72, 0xe5, 0xd4, 0x5c, 0x96, 0x00, 0xb1, 0x23, 0x73, 0x3f, 0xa2, 0xff, 0x73, 0x0e, - 0x16, 0x3d, 0x60, 0x76, 0x82, 0xae, 0x6e, 0x37, 0xe7, 0xe9, 0x76, 0x3f, 0x84, 0x25, 0x2b, 0x6d, - 0xa4, 0xe6, 0x62, 0x48, 0x3d, 0xac, 0x93, 0x2a, 0x37, 0x53, 0xfc, 0xab, 0x5d, 0xf9, 0x8c, 0x75, - 0x02, 0x0e, 0xb1, 0x6e, 0x31, 0xbe, 0x84, 0x5a, 0xaf, 0xf0, 0x4d, 0x9c, 0x06, 0x27, 0x13, 0x27, - 0x37, 0x23, 0x1d, 0x55, 0x30, 0xfb, 0x89, 0x4f, 0x90, 0xfd, 0x44, 0xb8, 0xb9, 0xc4, 0x44, 0x11, - 0xf1, 0xe4, 0x17, 0x7b, 0x75, 0x70, 0x79, 0xd3, 0xf0, 0xf5, 0xbe, 0x5b, 0x89, 0x47, 0xa6, 0x5b, - 0xa9, 0x6f, 0x0a, 0xdc, 0xcf, 0x2e, 0x2b, 0x59, 0xce, 0xd3, 0xa0, 0xec, 0x02, 0x49, 0xd2, 0xf6, - 0xff, 0x70, 0x30, 0x53, 0x6e, 0x61, 0xd5, 0xa4, 0x6b, 0x9b, 0xf7, 0xfc, 0x9f, 0x05, 0x5a, 0x8b, - 0xf8, 0xf7, 0x0b, 0xb2, 0x30, 0x7e, 0x7d, 0xe8, 0x3f, 0x67, 0x08, 0x53, 0xe8, 0xc4, 0xf5, 0x3f, - 0x22, 0x9e, 0x76, 0xc6, 0x07, 0x01, 0xca, 0x10, 0x3f, 0xc7, 0xdf, 0x1e, 0x81, 0xe5, 0xcc, 0xf3, - 0x04, 0x92, 0xe4, 0x45, 0x3d, 0x5a, 0x72, 0x5e, 0xf5, 0xbb, 0x1e, 0xdc, 0xf3, 0xcb, 0x3e, 0xa8, - 0x4d, 0xb7, 0xfd, 0x8f, 0xb3, 0x00, 0x83, 0x5c, 0x13, 0xbd, 0x80, 0x39, 0xf7, 0xa3, 0x5e, 0xb4, - 0x3a, 0xe4, 0x49, 0x39, 0xbf, 0x16, 0x3e, 0xe8, 0xc8, 0xf4, 0x02, 0xe6, 0xdc, 0x0f, 0xb9, 0x06, - 0xcc, 0x42, 0x1e, 0x93, 0x0d, 0x98, 0x85, 0xbe, 0xfd, 0x9a, 0x42, 0x1d, 0xb8, 0x16, 0xf1, 0x94, - 0x07, 0xdd, 0x19, 0xef, 0xc1, 0x13, 0xff, 0xbd, 0x31, 0xdf, 0x04, 0x09, 0x53, 0x48, 0x87, 0xeb, - 0x91, 0x2f, 0x58, 0xd0, 0xd6, 0xb8, 0xef, 0x6b, 0xf8, 0xbb, 0x63, 0x60, 0x3a, 0x73, 0xf6, 0x81, - 0x8f, 0x6e, 0x9b, 0xa3, 0xbb, 0x63, 0xbf, 0xe7, 0xe0, 0xef, 0x8d, 0xdf, 0x85, 0x17, 0xa6, 0xd0, - 0x3e, 0xa4, 0x5d, 0xfd, 0x53, 0xc4, 0x87, 0x36, 0x55, 0x29, 0xe3, 0xd5, 0x21, 0x0d, 0x57, 0xca, - 0xc9, 0xd5, 0xd2, 0x1a, 0x70, 0x0a, 0x36, 0xe7, 0x06, 0x9c, 0x42, 0x7a, 0x60, 0xfe, 0xed, 0xf7, - 0x5d, 0xf2, 0x61, 0xdb, 0x1f, 0x1e, 0x25, 0x84, 0x6d, 0x7f, 0x44, 0xc4, 0x20, 0x4c, 0xa1, 0xcf, - 0x60, 0xc1, 0x5b, 0xa6, 0x46, 0xeb, 0x43, 0xcb, 0xed, 0xfc, 0x8d, 0xa8, 0x61, 0x37, 0x4b, 0x6f, - 0x55, 0x74, 0xc0, 0x32, 0xb4, 0x3a, 0x3b, 0x60, 0x19, 0x51, 0x4c, 0x9d, 0xb2, 0xfc, 0x93, 0xa7, - 0xd6, 0x37, 0xf0, 0x4f, 0x61, 0x25, 0xca, 0x81, 0x7f, 0x0a, 0x2d, 0x10, 0x0a, 0x53, 0x48, 0x81, - 0x95, 0xf0, 0x52, 0x13, 0xba, 0x3d, 0x56, 0x25, 0x8d, 0xbf, 0x33, 0x0a, 0xcd, 0x99, 0xaa, 0x09, - 0x8b, 0x21, 0xed, 0x6d, 0x24, 0x0c, 0xed, 0x7d, 0xd3, 0x49, 0x6e, 0x8d, 0xd1, 0x1f, 0x17, 0xac, - 0x68, 0x63, 0xfb, 0xdf, 0x92, 0x90, 0x20, 0xd7, 0x7e, 0x03, 0xae, 0xf8, 0xea, 0x09, 0xe8, 0xc6, - 0xf0, 0x2a, 0x0b, 0xbf, 0x11, 0x39, 0xee, 0xac, 0xe1, 0x0d, 0x5c, 0x0d, 0x54, 0x08, 0xd0, 0xa6, - 0x9b, 0x2e, 0xac, 0x4a, 0xc1, 0xdf, 0x1c, 0x82, 0xe1, 0xe7, 0xed, 0xf5, 0x6d, 0x9b, 0xa3, 0x52, - 0x58, 0x2f, 0xef, 0x28, 0x7f, 0xf6, 0x05, 0x8d, 0xb2, 0xfc, 0x9e, 0x4c, 0xf0, 0xca, 0x15, 0xea, - 0xc3, 0x6e, 0x0d, 0xc5, 0x71, 0x66, 0xf8, 0xdc, 0x09, 0xef, 0x5c, 0x19, 0x14, 0xf2, 0x08, 0x17, - 0x9a, 0xe9, 0xf1, 0xc2, 0x30, 0x14, 0x87, 0xfd, 0x2b, 0xc8, 0xf8, 0xef, 0x79, 0xb4, 0x31, 0x22, - 0xec, 0xe0, 0x37, 0xa3, 0x11, 0xfc, 0x3b, 0xe3, 0x77, 0x32, 0x7e, 0xa9, 0xc2, 0xdc, 0xcb, 0xad, - 0xa1, 0x38, 0x6e, 0xb7, 0xe8, 0x8a, 0x70, 0x07, 0x6e, 0x31, 0x18, 0x0d, 0x0f, 0xdc, 0x62, 0x48, - 0x48, 0x2c, 0x4c, 0xed, 0x3c, 0x03, 0x90, 0x3b, 0xbd, 0xb7, 0xb2, 0x84, 0xd5, 0x7e, 0x17, 0xad, - 0x05, 0x3a, 0x50, 0x25, 0xb5, 0xdf, 0xad, 0xf5, 0xac, 0xcc, 0xcb, 0xc8, 0xfe, 0x7c, 0x86, 0xe4, - 0x5b, 0xb3, 0x84, 0xc0, 0x1a, 0xd8, 0xa9, 0x40, 0x66, 0x40, 0x2d, 0x91, 0x10, 0x0a, 0xdd, 0x0c, - 0xe5, 0x41, 0xfa, 0xf9, 0x3e, 0x46, 0x0b, 0x0e, 0x23, 0x32, 0xba, 0xf3, 0x09, 0x40, 0xd3, 0x50, - 0x24, 0x1a, 0xc3, 0xa1, 0xf5, 0x00, 0x9f, 0xe7, 0x0a, 0xee, 0xb4, 0x6c, 0x1e, 0x7f, 0xc1, 0x84, - 0x69, 0x1a, 0x0a, 0x8d, 0xf4, 0x76, 0x7e, 0x04, 0x69, 0x2a, 0xcc, 0x89, 0x85, 0x37, 0x8a, 0x9e, - 0xc9, 0x40, 0x57, 0x4f, 0x46, 0x76, 0x4a, 0x30, 0x4f, 0x19, 0xb0, 0xac, 0x11, 0x6d, 0x04, 0x58, - 0x1c, 0xd0, 0x11, 0x1f, 0x93, 0x39, 0x42, 0xc6, 0xc6, 0x76, 0x0a, 0x30, 0x67, 0xb3, 0x31, 0xdf, - 0x6a, 0x2d, 0x74, 0x23, 0x84, 0x8b, 0x35, 0xe0, 0x63, 0x92, 0x66, 0x4c, 0xac, 0xa1, 0x81, 0x28, - 0xf6, 0x3f, 0x9b, 0x06, 0x45, 0x61, 0x99, 0x5d, 0xa8, 0x28, 0x6c, 0xac, 0x90, 0x7c, 0x13, 0x6f, - 0x1a, 0xca, 0x71, 0x8a, 0x10, 0xfd, 0xe0, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x5e, 0xa7, 0xda, - 0x94, 0x19, 0x3d, 0x00, 0x00, + // 3802 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x3b, 0x4b, 0x6c, 0x23, 0x47, + 0x76, 0x6a, 0xfe, 0x24, 0x3d, 0x4a, 0x1a, 0xaa, 0x28, 0x69, 0x38, 0x2d, 0x69, 0xa4, 0xe9, 0xf1, + 0x78, 0xe5, 0xf1, 0x0c, 0x67, 0xad, 0xb5, 0x8d, 0x58, 0x1e, 0xef, 0x9a, 0xa4, 0x38, 0x12, 0x77, + 0x28, 0x52, 0x6e, 0x52, 0x33, 0x3b, 0x93, 0x18, 0xed, 0x16, 0x59, 0xe2, 0x34, 0x4c, 0x76, 0xd3, + 0xdd, 0x4d, 0x45, 0xf2, 0x25, 0x41, 0x82, 0x1c, 0x82, 0x5c, 0x72, 0x5b, 0xe7, 0xb6, 0x48, 0xf6, + 0x98, 0xc5, 0x22, 0x08, 0x82, 0x1c, 0x03, 0xe4, 0x18, 0x20, 0x9b, 0xdc, 0x12, 0xe4, 0xb2, 0x87, + 0x00, 0x39, 0x2c, 0x12, 0xc0, 0x97, 0x5c, 0x72, 0x08, 0x82, 0xae, 0xaa, 0x6e, 0xf6, 0x97, 0x9f, + 0x91, 0x06, 0x3e, 0xe4, 0x24, 0xf6, 0xab, 0xf7, 0x5e, 0xbd, 0xaa, 0x7a, 0xef, 0xd5, 0xfb, 0x94, + 0xe0, 0x83, 0x8e, 0x62, 0xbe, 0x1a, 0x9c, 0xe6, 0x5b, 0x5a, 0xef, 0x51, 0x4b, 0x53, 0x4d, 0x59, + 0x51, 0xb1, 0xfe, 0xd0, 0x30, 0x35, 0x5d, 0xee, 0xe0, 0x87, 0x8a, 0x6a, 0x62, 0xfd, 0x4c, 0x6e, + 0xe1, 0x47, 0x46, 0x1f, 0xb7, 0x1e, 0xb5, 0x0c, 0x25, 0xdf, 0xd7, 0x35, 0x53, 0x43, 0x29, 0xeb, + 0xe7, 0xf9, 0x7b, 0xfc, 0x76, 0x47, 0xd3, 0x3a, 0x5d, 0xfc, 0x88, 0x40, 0x4f, 0x07, 0x67, 0x8f, + 0xda, 0xd8, 0x68, 0xe9, 0x4a, 0xdf, 0xd4, 0x74, 0x8a, 0xc9, 0x6f, 0xf9, 0x31, 0x4c, 0xa5, 0x87, + 0x0d, 0x53, 0xee, 0xf5, 0x19, 0xc2, 0x6d, 0x3f, 0xc2, 0xef, 0xea, 0x72, 0xbf, 0x8f, 0x75, 0x83, + 0x8e, 0x0b, 0x6b, 0xb0, 0x72, 0x80, 0xcd, 0xe3, 0xee, 0xa0, 0xa3, 0xa8, 0x15, 0xf5, 0x4c, 0x13, + 0xf1, 0x57, 0x03, 0x6c, 0x98, 0xc2, 0xbf, 0x72, 0xb0, 0xea, 0x1b, 0x30, 0xfa, 0x9a, 0x6a, 0x60, + 0x84, 0x20, 0xa1, 0xca, 0x3d, 0x9c, 0xe3, 0xb6, 0xb9, 0x9d, 0x79, 0x91, 0xfc, 0x46, 0xf7, 0x60, + 0xe9, 0x1c, 0xab, 0x6d, 0x4d, 0x97, 0xce, 0xb1, 0x6e, 0x28, 0x9a, 0x9a, 0x8b, 0x91, 0xd1, 0x45, + 0x0a, 0x7d, 0x46, 0x81, 0xe8, 0x00, 0xe6, 0x7a, 0xb2, 0xaa, 0x9c, 0x61, 0xc3, 0xcc, 0xc5, 0xb7, + 0xe3, 0x3b, 0xe9, 0xdd, 0x77, 0xf3, 0x74, 0xa9, 0xf9, 0xd0, 0xb9, 0xf2, 0x47, 0x0c, 0xbb, 0xac, + 0x9a, 0xfa, 0xa5, 0xe8, 0x10, 0xf3, 0x1f, 0xc3, 0xa2, 0x67, 0x08, 0x65, 0x20, 0xfe, 0x25, 0xbe, + 0x64, 0x32, 0x59, 0x3f, 0xd1, 0x0a, 0x24, 0xcf, 0xe5, 0xee, 0x00, 0x33, 0x49, 0xe8, 0xc7, 0x5e, + 0xec, 0xb7, 0x38, 0xe1, 0x36, 0x6c, 0x38, 0xb3, 0x95, 0xe4, 0xbe, 0x7c, 0xaa, 0x74, 0x15, 0x53, + 0xc1, 0x86, 0xbd, 0xf4, 0xcf, 0x61, 0x33, 0x62, 0x9c, 0xed, 0xc0, 0x63, 0x58, 0x68, 0xb9, 0xe0, + 0x39, 0x8e, 0x2c, 0x25, 0x67, 0x2f, 0xc5, 0x47, 0x79, 0x29, 0x7a, 0xb0, 0x85, 0x5f, 0xc5, 0x21, + 0xe3, 0x47, 0x41, 0x8f, 0x61, 0xd6, 0xc0, 0xfa, 0xb9, 0xd2, 0xa2, 0xfb, 0x9a, 0xde, 0xdd, 0x8e, + 0xe2, 0x96, 0x6f, 0x50, 0xbc, 0xc3, 0x19, 0xd1, 0x26, 0x41, 0x27, 0x90, 0x39, 0xd7, 0xba, 0x83, + 0x1e, 0x96, 0xf0, 0x45, 0x5f, 0x56, 0x9d, 0x03, 0x48, 0xef, 0xee, 0x44, 0xb2, 0x79, 0x46, 0x08, + 0xca, 0x36, 0xfe, 0xe1, 0x8c, 0x78, 0xe3, 0xdc, 0x0b, 0xe2, 0x7f, 0xca, 0xc1, 0x2c, 0x9b, 0x0d, + 0x7d, 0x04, 0x09, 0xf3, 0xb2, 0x4f, 0xa5, 0x5b, 0xda, 0xbd, 0x37, 0x4e, 0xba, 0x7c, 0xf3, 0xb2, + 0x8f, 0x45, 0x42, 0x22, 0x7c, 0x06, 0x09, 0xeb, 0x0b, 0xa5, 0x61, 0xf6, 0xa4, 0xf6, 0xb4, 0x56, + 0x7f, 0x5e, 0xcb, 0xcc, 0xa0, 0x35, 0x40, 0xa5, 0x7a, 0xad, 0x29, 0xd6, 0xab, 0xd5, 0xb2, 0x28, + 0x35, 0xca, 0xe2, 0xb3, 0x4a, 0xa9, 0x9c, 0xe1, 0xd0, 0x5b, 0xb0, 0xfd, 0xac, 0x5e, 0x3d, 0x39, + 0x2a, 0x4b, 0x85, 0x52, 0xa9, 0xdc, 0x68, 0x54, 0x8a, 0x95, 0x6a, 0xa5, 0xf9, 0x42, 0x2a, 0xd5, + 0x6b, 0x8d, 0xa6, 0x58, 0xa8, 0xd4, 0x9a, 0x8d, 0x4c, 0x8c, 0xff, 0x03, 0x0e, 0x6e, 0xf8, 0x16, + 0x80, 0x0a, 0x1e, 0x09, 0x1f, 0x4e, 0xba, 0x70, 0xb7, 0xa4, 0x0f, 0xc2, 0x24, 0x05, 0x48, 0xd5, + 0x6b, 0xd5, 0x4a, 0xcd, 0x92, 0x2e, 0x0d, 0xb3, 0xf5, 0x27, 0x4f, 0xc8, 0x47, 0xac, 0x98, 0xa2, + 0x13, 0x0a, 0x4b, 0xb0, 0x70, 0xac, 0x6b, 0xa7, 0xd8, 0xd6, 0x9f, 0x02, 0x2c, 0xb2, 0x6f, 0xa6, + 0x2f, 0xdf, 0x87, 0xa4, 0x8e, 0xe5, 0xf6, 0x25, 0x3b, 0x5a, 0x3e, 0x4f, 0x6d, 0x32, 0x6f, 0xdb, + 0x64, 0xbe, 0xa8, 0x69, 0xdd, 0x67, 0x96, 0x7e, 0x8a, 0x14, 0x51, 0xf8, 0x36, 0x01, 0xd9, 0x92, + 0x8e, 0x65, 0x13, 0x53, 0x69, 0x19, 0xeb, 0x50, 0xdb, 0x7b, 0x0c, 0x4b, 0x96, 0x7e, 0xb5, 0x14, + 0xf3, 0x52, 0xd2, 0x65, 0xb5, 0x83, 0xd9, 0xd1, 0xaf, 0xda, 0x3b, 0x50, 0x62, 0xa3, 0xa2, 0x35, + 0x28, 0x2e, 0xb6, 0xdc, 0x9f, 0xa8, 0x02, 0x59, 0xa6, 0x3a, 0x1e, 0x95, 0x8e, 0x7b, 0x55, 0x9a, + 0x4a, 0xe1, 0x52, 0x69, 0x74, 0xee, 0x85, 0x28, 0xd8, 0x40, 0x4f, 0x01, 0xfa, 0xb2, 0x2e, 0xf7, + 0xb0, 0x89, 0x75, 0x23, 0x97, 0xf0, 0xda, 0x77, 0xc8, 0x6a, 0xf2, 0xc7, 0x0e, 0x36, 0xb5, 0x6f, + 0x17, 0x39, 0x3a, 0xb0, 0x0c, 0xa2, 0xa5, 0x63, 0xd3, 0xc8, 0x25, 0x09, 0xa7, 0x9d, 0x51, 0x9c, + 0x1a, 0x14, 0x95, 0xb0, 0x29, 0xc6, 0xbf, 0x29, 0x72, 0xa2, 0x4d, 0x8d, 0xea, 0xb0, 0x6a, 0x2f, + 0x50, 0x53, 0x4d, 0xac, 0x9a, 0x92, 0xa1, 0x0d, 0xf4, 0x16, 0xce, 0xa5, 0xc8, 0x2e, 0xad, 0xfb, + 0x96, 0x48, 0x71, 0x1a, 0x04, 0x45, 0x64, 0x5b, 0xe3, 0x01, 0xa2, 0x97, 0xc0, 0xcb, 0xad, 0x16, + 0x36, 0x0c, 0x85, 0xee, 0x85, 0xa4, 0xe3, 0xaf, 0x06, 0x8a, 0x8e, 0x7b, 0x58, 0x35, 0x8d, 0xdc, + 0xac, 0x97, 0x6b, 0x53, 0xeb, 0x6b, 0x5d, 0xad, 0x73, 0x29, 0x0e, 0x71, 0xc4, 0x5b, 0x1e, 0x72, + 0xd7, 0x88, 0xc1, 0x7f, 0x02, 0x37, 0x7c, 0x9b, 0x32, 0x8d, 0x67, 0xe3, 0xf7, 0x60, 0xc1, 0xbd, + 0x13, 0x53, 0x79, 0xc5, 0x3f, 0x89, 0x41, 0x36, 0x64, 0x0f, 0xd0, 0x21, 0xcc, 0x19, 0xaa, 0xdc, + 0x37, 0x5e, 0x69, 0x26, 0xd3, 0xdf, 0xfb, 0x23, 0xb6, 0x2c, 0xdf, 0x60, 0xb8, 0xf4, 0xf3, 0x70, + 0x46, 0x74, 0xa8, 0x51, 0x11, 0x52, 0x74, 0x3f, 0xfd, 0xbe, 0x29, 0x8c, 0x0f, 0x85, 0x39, 0x5c, + 0x18, 0x25, 0xff, 0x1e, 0x2c, 0x79, 0x67, 0x40, 0x5b, 0x90, 0xb6, 0x67, 0x90, 0x94, 0x36, 0x5b, + 0x2b, 0xd8, 0xa0, 0x4a, 0x9b, 0x7f, 0x17, 0x16, 0xdc, 0xcc, 0xd0, 0x3a, 0xcc, 0x33, 0x85, 0x70, + 0xd0, 0xe7, 0x28, 0xa0, 0xd2, 0x76, 0x6c, 0xfa, 0x87, 0xb0, 0xe2, 0xd5, 0x33, 0x66, 0xca, 0x6f, + 0x3b, 0x6b, 0xa0, 0x7b, 0xb1, 0xe4, 0x5d, 0x83, 0x2d, 0xa7, 0xf0, 0xf3, 0x24, 0x64, 0xfc, 0x46, + 0x83, 0x1e, 0x43, 0xf2, 0xb4, 0xab, 0xb5, 0xbe, 0x64, 0xb4, 0x6f, 0x45, 0x59, 0x57, 0xbe, 0x68, + 0x61, 0x51, 0xe8, 0xe1, 0x8c, 0x48, 0x89, 0x2c, 0xea, 0x9e, 0x36, 0x50, 0x4d, 0xb6, 0x7b, 0xd1, + 0xd4, 0x47, 0x16, 0xd6, 0x90, 0x9a, 0x10, 0xa1, 0x7d, 0x48, 0x53, 0xb5, 0x93, 0x7a, 0x5a, 0x1b, + 0xe7, 0xe2, 0x84, 0xc7, 0xdd, 0x48, 0x1e, 0x05, 0x82, 0x7b, 0xa4, 0xb5, 0xb1, 0x08, 0xb2, 0xf3, + 0x9b, 0x5f, 0x84, 0xb4, 0x4b, 0x36, 0xfe, 0x6b, 0x48, 0xbb, 0x26, 0x43, 0x37, 0x61, 0xf6, 0xcc, + 0x90, 0x1c, 0x27, 0x3c, 0x2f, 0xa6, 0xce, 0x0c, 0xe2, 0x4f, 0xb7, 0x20, 0x4d, 0xa4, 0x90, 0xce, + 0xba, 0x72, 0xc7, 0xc8, 0xc5, 0xb6, 0xe3, 0xd6, 0x19, 0x11, 0xd0, 0x13, 0x0b, 0x82, 0xde, 0x03, + 0xe6, 0x50, 0x24, 0x8a, 0xd7, 0xd1, 0xb5, 0x41, 0x9f, 0x08, 0x39, 0x5f, 0x8c, 0xff, 0xac, 0xc8, + 0x89, 0xec, 0x7e, 0x23, 0xb3, 0x1d, 0x58, 0x83, 0xfc, 0x5f, 0xc7, 0x00, 0x86, 0x52, 0xa2, 0xc7, + 0x90, 0x20, 0x0b, 0xa3, 0xde, 0x7f, 0x67, 0x82, 0x85, 0xe5, 0xc9, 0xea, 0x08, 0x95, 0xf0, 0x1f, + 0x1c, 0x24, 0x08, 0x1b, 0xff, 0x1d, 0xd5, 0xa8, 0xd4, 0x0e, 0xaa, 0x65, 0xa9, 0x56, 0xdf, 0x2f, + 0x4b, 0xcf, 0xc5, 0x4a, 0xb3, 0x2c, 0x66, 0x38, 0xb4, 0x0e, 0x37, 0xdd, 0x70, 0xb1, 0x5c, 0xd8, + 0x2f, 0x8b, 0x52, 0xbd, 0x56, 0x7d, 0x91, 0x89, 0x21, 0x1e, 0xd6, 0x8e, 0x4e, 0xaa, 0xcd, 0x4a, + 0x70, 0x2c, 0x8e, 0x36, 0x20, 0xe7, 0x1a, 0x63, 0x3c, 0x18, 0xdb, 0x84, 0xc5, 0xd6, 0x35, 0x4a, + 0x7f, 0xb2, 0xc1, 0x24, 0x12, 0xe0, 0x96, 0x7b, 0x4e, 0x2f, 0x6d, 0x8a, 0xb7, 0x36, 0x09, 0xdd, + 0x81, 0x9c, 0x1b, 0xc7, 0xc3, 0x61, 0x96, 0xa0, 0x14, 0x17, 0x1d, 0x35, 0x20, 0x6a, 0xfe, 0x1c, + 0x16, 0x3d, 0xb7, 0x83, 0x15, 0xc8, 0x31, 0x77, 0xd6, 0x96, 0x4e, 0x2f, 0x4d, 0x12, 0xdc, 0x70, + 0x3b, 0x71, 0x71, 0xd1, 0x86, 0x16, 0x2d, 0xa0, 0x75, 0xa0, 0x5d, 0xa5, 0xa7, 0x98, 0x0c, 0x27, + 0x46, 0x70, 0x80, 0x80, 0x08, 0x82, 0xf0, 0xeb, 0x18, 0xa4, 0x98, 0x56, 0xdc, 0x73, 0xdd, 0x4f, + 0x1e, 0x96, 0x36, 0x94, 0xb2, 0xf4, 0x98, 0x65, 0xcc, 0x6b, 0x96, 0xe8, 0x10, 0x96, 0xdc, 0x4e, + 0xfc, 0xc2, 0x0e, 0x1f, 0xef, 0x78, 0xcf, 0xd9, 0xed, 0x49, 0x2e, 0x58, 0xd0, 0xb8, 0x78, 0xee, + 0x86, 0xa1, 0x22, 0x2c, 0xf9, 0xee, 0x81, 0xc4, 0xf8, 0x7b, 0x60, 0xb1, 0xe5, 0x71, 0x89, 0x05, + 0xc8, 0xda, 0x2e, 0xbc, 0x8b, 0x25, 0x93, 0xb9, 0x78, 0x76, 0x4f, 0x65, 0x02, 0xae, 0x1f, 0x0d, + 0x91, 0x6d, 0x18, 0xff, 0x29, 0xa0, 0xa0, 0xac, 0x53, 0xf9, 0xeb, 0x01, 0x64, 0x43, 0x2e, 0x17, + 0x94, 0x87, 0x79, 0x72, 0x54, 0x86, 0x62, 0x62, 0x16, 0x98, 0x06, 0x25, 0x1a, 0xa2, 0x58, 0xf8, + 0x7d, 0x1d, 0x9f, 0x61, 0x5d, 0xc7, 0x6d, 0x62, 0x98, 0xa1, 0xf8, 0x0e, 0x8a, 0xf0, 0x87, 0x1c, + 0xcc, 0xd9, 0x70, 0xb4, 0x07, 0x73, 0x06, 0xee, 0xd0, 0x8b, 0x8f, 0xce, 0x75, 0xdb, 0x4f, 0x9b, + 0x6f, 0x30, 0x04, 0x16, 0xc2, 0xdb, 0xf8, 0x56, 0x08, 0xef, 0x19, 0x9a, 0x6a, 0xf1, 0x7f, 0xcb, + 0x41, 0x76, 0x1f, 0x77, 0xb1, 0x3f, 0x3e, 0x1a, 0xe5, 0xdb, 0xdd, 0x21, 0x45, 0xcc, 0x1b, 0x52, + 0x84, 0xb0, 0x1a, 0x11, 0x52, 0x5c, 0xe9, 0x9a, 0x5d, 0x83, 0x15, 0xef, 0x6c, 0xf4, 0x62, 0x11, + 0xfe, 0x2b, 0x0e, 0xb7, 0x2d, 0x5d, 0xd0, 0xb5, 0x6e, 0x17, 0xeb, 0xc7, 0x83, 0xd3, 0xae, 0x62, + 0xbc, 0x9a, 0x62, 0x71, 0x37, 0x61, 0x56, 0xd5, 0xda, 0x2e, 0xe3, 0x49, 0x59, 0x9f, 0x95, 0x36, + 0x2a, 0xc3, 0xb2, 0x3f, 0xc0, 0xbb, 0x64, 0xee, 0x3f, 0x3a, 0xbc, 0xcb, 0x9c, 0xfb, 0xef, 0x2e, + 0x1e, 0xe6, 0xac, 0xd0, 0x54, 0x53, 0xbb, 0x97, 0xc4, 0x62, 0xe6, 0x44, 0xe7, 0x1b, 0x89, 0xfe, + 0x58, 0xed, 0x07, 0x4e, 0xac, 0x36, 0x72, 0x45, 0xa3, 0xc2, 0xb6, 0x2f, 0x02, 0x16, 0x9f, 0x22, + 0xac, 0x3f, 0x9a, 0x90, 0xf5, 0x58, 0x4f, 0x70, 0x95, 0x53, 0xbc, 0x06, 0xf3, 0xfd, 0x07, 0x0e, + 0xb6, 0x22, 0x97, 0xc0, 0x82, 0x8d, 0x36, 0xdc, 0xe8, 0xd3, 0x01, 0x67, 0x13, 0xa8, 0x95, 0x7d, + 0x3c, 0x76, 0x13, 0x58, 0xfe, 0xcc, 0xa0, 0x9e, 0x6d, 0x58, 0xea, 0x7b, 0x80, 0x7c, 0x01, 0xb2, + 0x21, 0x68, 0x53, 0x2d, 0xe6, 0x37, 0x1c, 0x6c, 0x0f, 0x45, 0x39, 0x51, 0xfb, 0xd7, 0xa7, 0xbe, + 0xcd, 0xa1, 0x6e, 0x51, 0x97, 0xff, 0x41, 0x70, 0xed, 0xe1, 0x13, 0xbe, 0x29, 0x0b, 0xbe, 0x0b, + 0x77, 0x46, 0x4c, 0xcd, 0xcc, 0xf9, 0xd7, 0x09, 0xb8, 0xf3, 0x4c, 0xee, 0x2a, 0x6d, 0x27, 0x84, + 0x0c, 0xa9, 0x34, 0x8c, 0xde, 0x92, 0x56, 0xc0, 0x02, 0xa8, 0xd7, 0x7a, 0xec, 0x58, 0xed, 0x38, + 0xfe, 0x13, 0x5c, 0x87, 0xd7, 0x98, 0xfe, 0xbd, 0x08, 0x49, 0xff, 0x3e, 0x9a, 0x5c, 0xd6, 0x51, + 0xc9, 0xe0, 0x89, 0xdf, 0xc1, 0x7c, 0x38, 0x39, 0xdf, 0x11, 0x5a, 0x70, 0x65, 0x2b, 0xfe, 0x2e, + 0xf3, 0xb5, 0xbf, 0x4f, 0x80, 0x30, 0x6a, 0xf5, 0xcc, 0x87, 0x88, 0x30, 0xdf, 0xd2, 0xd4, 0x33, + 0x45, 0xef, 0xe1, 0x36, 0xcb, 0x3b, 0xde, 0x9f, 0x64, 0xf3, 0x98, 0x03, 0x29, 0xd9, 0xb4, 0xe2, + 0x90, 0x0d, 0xca, 0xc1, 0x6c, 0x0f, 0x1b, 0x86, 0xdc, 0xb1, 0xc5, 0xb2, 0x3f, 0xf9, 0x5f, 0xc4, + 0x61, 0xde, 0x21, 0x41, 0x6a, 0x40, 0x83, 0xa9, 0xfb, 0x3a, 0x78, 0x1d, 0x01, 0x5e, 0x5f, 0x99, + 0x63, 0xaf, 0xa1, 0xcc, 0x6d, 0x8f, 0x32, 0x53, 0x73, 0xd8, 0x7f, 0x2d, 0xb1, 0x47, 0xe8, 0xf5, + 0x77, 0xae, 0x80, 0xc2, 0xef, 0x00, 0xaa, 0x2a, 0x06, 0xcb, 0xdf, 0x1c, 0xb7, 0x64, 0xa5, 0x6b, + 0xf2, 0x85, 0x84, 0x55, 0x53, 0x57, 0x58, 0xb8, 0x9e, 0x14, 0xa1, 0x27, 0x5f, 0x94, 0x29, 0xc4, + 0x0a, 0xe9, 0x0d, 0x53, 0xd6, 0x4d, 0x45, 0xed, 0x48, 0xa6, 0xf6, 0x25, 0x76, 0xca, 0xbd, 0x36, + 0xb4, 0x69, 0x01, 0x85, 0xff, 0x8c, 0x41, 0xd6, 0xc3, 0x9e, 0xe9, 0xe4, 0xc7, 0x30, 0x3b, 0xe4, + 0xed, 0x09, 0xe3, 0x43, 0xb0, 0xf3, 0x74, 0xdb, 0x6c, 0x0a, 0xb4, 0x09, 0xa0, 0xe2, 0x0b, 0xd3, + 0x33, 0xef, 0xbc, 0x05, 0x21, 0x73, 0xf2, 0x7f, 0xc4, 0x39, 0xe9, 0xbe, 0x29, 0x9b, 0x03, 0x03, + 0x3d, 0x00, 0xc4, 0x5c, 0x34, 0x6e, 0x4b, 0xec, 0x8e, 0xa1, 0xf3, 0xce, 0x8b, 0x19, 0x67, 0xa4, + 0x46, 0x6e, 0x1b, 0x03, 0x1d, 0x38, 0x95, 0xd4, 0x96, 0xa6, 0xb6, 0x15, 0x73, 0x58, 0x49, 0xbd, + 0x19, 0x48, 0x10, 0xe8, 0x30, 0xcd, 0x4f, 0x6f, 0x9c, 0x7b, 0xa1, 0xfc, 0x57, 0x90, 0xa4, 0xc7, + 0x31, 0x61, 0xc5, 0x00, 0x7d, 0x0a, 0x29, 0x83, 0x48, 0xec, 0xaf, 0x8e, 0x84, 0xed, 0x89, 0x7b, + 0x85, 0x22, 0xa3, 0x13, 0x7e, 0x08, 0xfc, 0xf0, 0x62, 0x3a, 0xc0, 0xe6, 0xe4, 0xd7, 0xef, 0x9e, + 0xb5, 0x06, 0xe1, 0xa7, 0x31, 0x58, 0x0f, 0x65, 0x30, 0x5d, 0xed, 0x03, 0x1d, 0xfa, 0x56, 0xf2, + 0xfd, 0xe0, 0x8d, 0x1d, 0x60, 0x1e, 0xba, 0x22, 0xfe, 0xf7, 0xaf, 0x76, 0x98, 0xc5, 0xa9, 0x0f, + 0x33, 0x70, 0x8e, 0x74, 0x67, 0x7e, 0x11, 0x03, 0x74, 0x80, 0x4d, 0x27, 0x55, 0x66, 0x5b, 0x1a, + 0xe1, 0x6f, 0xb8, 0xd7, 0xf0, 0x37, 0x3f, 0xf6, 0xf8, 0x1b, 0xea, 0xb1, 0xee, 0xbb, 0x7a, 0x23, + 0xbe, 0xa9, 0x47, 0xde, 0x96, 0x11, 0xe9, 0x29, 0x8d, 0xf9, 0x27, 0x4b, 0x4f, 0xaf, 0xe8, 0x56, + 0xfe, 0x9d, 0x83, 0xac, 0x47, 0x68, 0xa6, 0x41, 0x0f, 0x01, 0xc9, 0xe7, 0xb2, 0xd2, 0x95, 0x2d, + 0xc1, 0xec, 0xf4, 0x9f, 0x95, 0x03, 0x96, 0x9d, 0x11, 0x9b, 0x0c, 0x3d, 0x85, 0x6c, 0x4f, 0xbe, + 0x50, 0x7a, 0x83, 0x9e, 0xc4, 0xf6, 0xd9, 0x50, 0xbe, 0xb6, 0xab, 0x87, 0xeb, 0x81, 0x2a, 0x7a, + 0x45, 0x35, 0x3f, 0x7c, 0x9f, 0x96, 0xd1, 0x97, 0x19, 0x1d, 0x53, 0x1e, 0xe5, 0x6b, 0x8c, 0x8e, + 0x21, 0xdb, 0x53, 0xd4, 0x00, 0xb3, 0xf8, 0x58, 0x66, 0xd4, 0xc0, 0x97, 0x19, 0xf1, 0x90, 0xa3, + 0x20, 0xb8, 0x83, 0x5e, 0xb6, 0x5c, 0x7f, 0x2f, 0xa9, 0xeb, 0x0e, 0x16, 0x03, 0x38, 0x6c, 0x5b, + 0x0e, 0x42, 0xfb, 0x49, 0x77, 0x83, 0x66, 0xc3, 0x9a, 0x2b, 0x91, 0xad, 0xa5, 0xff, 0x8d, 0xbb, + 0x2d, 0x38, 0x80, 0x8d, 0x3e, 0x86, 0xb8, 0xde, 0x6f, 0x31, 0xf3, 0xfd, 0xde, 0x04, 0xfc, 0xf3, + 0xe2, 0x71, 0xe9, 0x70, 0x46, 0xb4, 0xa8, 0xf8, 0x3f, 0x8b, 0x43, 0x5c, 0x3c, 0x2e, 0xa1, 0x4f, + 0x3d, 0x7d, 0x96, 0x07, 0x13, 0x72, 0x71, 0xb7, 0x59, 0xfe, 0x29, 0x16, 0xd6, 0x67, 0xc9, 0xc1, + 0x4a, 0x49, 0x2c, 0x17, 0x9a, 0x65, 0x69, 0xbf, 0x5c, 0x2d, 0x37, 0xcb, 0x12, 0xed, 0x03, 0x65, + 0x38, 0xb4, 0x01, 0xb9, 0xe3, 0x93, 0x62, 0xb5, 0xd2, 0x38, 0x94, 0x4e, 0x6a, 0xf6, 0x2f, 0x36, + 0x1a, 0x43, 0x19, 0x58, 0xa8, 0x56, 0x1a, 0x4d, 0x06, 0x68, 0x64, 0xe2, 0x16, 0xe4, 0xa0, 0xdc, + 0x94, 0x4a, 0x85, 0xe3, 0x42, 0xa9, 0xd2, 0x7c, 0x91, 0x49, 0x20, 0x1e, 0xd6, 0xbc, 0xbc, 0x1b, + 0xb5, 0xc2, 0x71, 0xe3, 0xb0, 0xde, 0xcc, 0x24, 0x11, 0x82, 0x25, 0x42, 0x6f, 0x83, 0x1a, 0x99, + 0x94, 0xc5, 0xa1, 0x54, 0xad, 0xd7, 0x1c, 0x19, 0x66, 0xd1, 0x0a, 0x64, 0xec, 0x99, 0xc5, 0x72, + 0x61, 0x9f, 0x14, 0xf4, 0xe6, 0xd0, 0x32, 0x2c, 0x96, 0x7f, 0x72, 0x5c, 0xa8, 0xed, 0xdb, 0x88, + 0xf3, 0x68, 0x1b, 0x36, 0xdc, 0xe2, 0x48, 0x8c, 0xaa, 0xbc, 0x4f, 0x8a, 0x72, 0x8d, 0x0c, 0xa0, + 0x5b, 0x90, 0x61, 0x2d, 0xae, 0x52, 0xbd, 0xb6, 0x5f, 0x69, 0x56, 0xea, 0xb5, 0x4c, 0x9a, 0x56, + 0xf0, 0xb2, 0x00, 0x96, 0xe4, 0x8c, 0xd9, 0xc2, 0xf8, 0xb2, 0xde, 0x22, 0x2d, 0xeb, 0xd9, 0x65, + 0xeb, 0xdf, 0xc4, 0x60, 0x95, 0xd6, 0xad, 0xed, 0x2a, 0xb9, 0xed, 0xab, 0x76, 0x20, 0x43, 0xeb, + 0x5d, 0x92, 0xff, 0x16, 0x58, 0xa2, 0xf0, 0x67, 0x76, 0xde, 0x61, 0xf7, 0x98, 0x62, 0xae, 0x1e, + 0x53, 0xc5, 0x9f, 0x85, 0xdd, 0xf7, 0x76, 0x63, 0x7c, 0xb3, 0x8d, 0x4a, 0xec, 0x8f, 0x42, 0xd2, + 0x84, 0x87, 0xa3, 0xb9, 0x8d, 0x0a, 0xa1, 0xae, 0x92, 0xc5, 0x5f, 0xd1, 0xcb, 0x3d, 0x81, 0x35, + 0xbf, 0xbc, 0xcc, 0xa0, 0x1f, 0x04, 0x7a, 0x26, 0x8e, 0xdb, 0x75, 0x70, 0x1d, 0x0c, 0xe1, 0x5f, + 0x38, 0x98, 0xb3, 0xc1, 0x56, 0x78, 0x63, 0xf9, 0x25, 0x4f, 0xa5, 0x74, 0xde, 0x82, 0x38, 0x85, + 0x57, 0x77, 0xb7, 0x23, 0xe6, 0xef, 0x76, 0x84, 0x9e, 0x73, 0x3c, 0xf4, 0x9c, 0x7f, 0x04, 0x8b, + 0x2d, 0x4b, 0x7c, 0x45, 0x53, 0x25, 0x53, 0xe9, 0xd9, 0x85, 0xd0, 0x60, 0x77, 0xb2, 0x69, 0x3f, + 0x29, 0x10, 0x17, 0x6c, 0x02, 0x0b, 0x84, 0xb6, 0x61, 0x81, 0x74, 0x2b, 0x25, 0x53, 0x93, 0x06, + 0x06, 0xce, 0x25, 0x49, 0x59, 0x08, 0x08, 0xac, 0xa9, 0x9d, 0x18, 0x58, 0xf8, 0x3b, 0x0e, 0x56, + 0x69, 0xb5, 0xcb, 0xaf, 0x8e, 0xe3, 0xba, 0x36, 0x6e, 0x8d, 0xf3, 0xdd, 0x86, 0xa1, 0x0c, 0xdf, + 0x54, 0xb2, 0x9f, 0x83, 0x35, 0xff, 0x7c, 0x2c, 0xc3, 0xff, 0x65, 0x0c, 0x56, 0xac, 0xd0, 0xcc, + 0x1e, 0xb8, 0xee, 0xe8, 0x79, 0x8a, 0x93, 0xf4, 0x6d, 0x66, 0x22, 0xb0, 0x99, 0x87, 0xfe, 0xfc, + 0xf9, 0x1d, 0x77, 0x70, 0xe9, 0x5f, 0xc1, 0x9b, 0xda, 0xcb, 0xbf, 0xe4, 0x60, 0xd5, 0x37, 0x1f, + 0xb3, 0x97, 0x4f, 0xfc, 0x09, 0xc1, 0xdd, 0x08, 0xf9, 0x5e, 0x2b, 0x25, 0xf8, 0xc0, 0x0e, 0xc5, + 0xa7, 0x33, 0xcb, 0x7f, 0x8e, 0xc1, 0xe6, 0xf0, 0x52, 0x23, 0xef, 0x05, 0xda, 0x53, 0x54, 0xb4, + 0xae, 0xd6, 0x96, 0xff, 0xcc, 0xef, 0x70, 0x77, 0x83, 0xf7, 0x6c, 0x88, 0x48, 0xa3, 0x1c, 0x6f, + 0x68, 0x21, 0x38, 0x31, 0x6d, 0x21, 0xf8, 0x4a, 0x1a, 0xf0, 0x7b, 0xee, 0x1a, 0xb7, 0x57, 0x7c, + 0xa6, 0x09, 0x13, 0x36, 0x8b, 0x3e, 0x84, 0x9b, 0x24, 0xfa, 0x77, 0x9e, 0xbb, 0xd8, 0x4d, 0x78, + 0xea, 0x12, 0xe7, 0xc4, 0x55, 0x6b, 0xd8, 0x79, 0xe3, 0xc1, 0x1a, 0x24, 0x6d, 0xe1, 0xdb, 0x04, + 0xac, 0x59, 0xd9, 0x41, 0xc3, 0x94, 0x3b, 0xd3, 0xb4, 0x0e, 0x7e, 0x3b, 0x58, 0x89, 0x8d, 0x79, + 0x8f, 0x25, 0x9c, 0xeb, 0x24, 0x05, 0x58, 0x94, 0x87, 0xac, 0x61, 0xca, 0x1d, 0xe2, 0x0e, 0x64, + 0xbd, 0x83, 0x4d, 0xa9, 0x2f, 0x9b, 0xaf, 0x98, 0xad, 0x2f, 0xb3, 0xa1, 0x26, 0x19, 0x39, 0x96, + 0xcd, 0x57, 0xd7, 0x74, 0x90, 0xe8, 0xc7, 0x7e, 0xa7, 0xf0, 0xee, 0x98, 0xb5, 0x8c, 0xd0, 0xad, + 0x9f, 0x44, 0x54, 0xeb, 0xdf, 0x1b, 0xc3, 0x72, 0x7c, 0x95, 0xfe, 0xea, 0xd5, 0xe9, 0xef, 0xb8, + 0xd0, 0x7f, 0x0b, 0x6e, 0x06, 0x16, 0xcf, 0xae, 0x90, 0x0e, 0xe4, 0xac, 0xa1, 0x13, 0xd5, 0x98, + 0x52, 0x1d, 0x23, 0x34, 0x26, 0x16, 0xa1, 0x31, 0xc2, 0x3a, 0xdc, 0x0a, 0x99, 0x88, 0x49, 0xf1, + 0x37, 0x49, 0x2a, 0xc6, 0xf4, 0x3d, 0xa7, 0xcf, 0xa3, 0xac, 0xe2, 0x7d, 0xf7, 0xb1, 0x87, 0xb6, + 0x67, 0xde, 0x84, 0x5d, 0x6c, 0x41, 0xda, 0x8d, 0xc7, 0xae, 0x41, 0x73, 0x8c, 0xe1, 0x24, 0xaf, + 0xd4, 0x0a, 0x4b, 0xf9, 0x5a, 0x61, 0xd5, 0xa1, 0x51, 0xcd, 0x7a, 0x43, 0xdb, 0xc8, 0xad, 0x18, + 0x61, 0x56, 0x2f, 0x03, 0x66, 0x35, 0xe7, 0xed, 0xaf, 0x45, 0x32, 0xfd, 0x7f, 0x60, 0x58, 0x4c, + 0xa9, 0x43, 0x1b, 0x5f, 0xc2, 0x4b, 0xe0, 0xa9, 0xc6, 0x4f, 0xdf, 0x8a, 0xf2, 0xa9, 0x51, 0xcc, + 0xaf, 0x46, 0xc2, 0x26, 0xac, 0x87, 0xf2, 0x66, 0x53, 0xff, 0x31, 0x47, 0x05, 0x73, 0x6a, 0x5c, + 0x0d, 0x53, 0x36, 0x8d, 0x49, 0xa7, 0x66, 0x83, 0xee, 0xa9, 0x29, 0x88, 0x68, 0xf0, 0x94, 0x26, + 0x21, 0xfc, 0x29, 0x47, 0xf7, 0xc1, 0x2f, 0x0b, 0xbb, 0x6d, 0xdf, 0x81, 0xe4, 0x80, 0x94, 0xf1, + 0x69, 0xd4, 0x95, 0xf5, 0x1a, 0xc1, 0x89, 0x35, 0x24, 0x52, 0x8c, 0x6b, 0x2b, 0x8c, 0x0a, 0xbf, + 0xe4, 0x20, 0xed, 0xe2, 0x8f, 0x36, 0x60, 0xde, 0xa9, 0xfc, 0xd8, 0xf9, 0x8e, 0x03, 0xb0, 0x8e, + 0xdf, 0xd4, 0x4c, 0xb9, 0xcb, 0x9e, 0x98, 0xd0, 0x0f, 0x2b, 0x45, 0x1d, 0x18, 0x98, 0x86, 0xc3, + 0x71, 0x91, 0xfc, 0x46, 0x0f, 0x20, 0x31, 0x50, 0x15, 0x93, 0x98, 0xfd, 0x92, 0xdf, 0x9e, 0xc9, + 0x54, 0xf9, 0x13, 0x55, 0x31, 0x45, 0x82, 0x25, 0xdc, 0x87, 0x84, 0xf5, 0xe5, 0xad, 0x40, 0xcc, + 0x43, 0xb2, 0xf8, 0xa2, 0x59, 0x6e, 0x64, 0x38, 0x04, 0x90, 0xaa, 0xd0, 0x7c, 0x3d, 0x26, 0x54, + 0xed, 0xb7, 0xa6, 0xce, 0x22, 0x2c, 0x17, 0x20, 0x9f, 0xaa, 0x9a, 0xde, 0x93, 0xbb, 0x44, 0xe6, + 0x39, 0xd1, 0xf9, 0x8e, 0xee, 0x8e, 0xd0, 0x5a, 0xe2, 0x86, 0x73, 0x22, 0x61, 0xf5, 0xa2, 0x2f, + 0xa8, 0x6e, 0x45, 0x55, 0x8a, 0x0a, 0xa1, 0x95, 0xa2, 0x4d, 0xcf, 0x2d, 0x3b, 0xa6, 0x46, 0xf4, + 0xab, 0x18, 0xac, 0x86, 0xe2, 0xa1, 0x0f, 0xdc, 0xd5, 0xa1, 0x3b, 0x23, 0x79, 0xba, 0xeb, 0x42, + 0xff, 0xcd, 0xd1, 0xba, 0xd0, 0x9e, 0xa7, 0x2e, 0xf4, 0xf6, 0x58, 0x7a, 0x77, 0x45, 0xe8, 0xaf, + 0xb8, 0x88, 0x8a, 0x50, 0xa3, 0x59, 0x38, 0x28, 0x4b, 0x27, 0x35, 0xfa, 0xd7, 0xa9, 0x08, 0xad, + 0x40, 0x66, 0x58, 0x27, 0x91, 0x1a, 0xcd, 0x42, 0xb3, 0x91, 0x89, 0x05, 0xab, 0x31, 0xf1, 0xd0, + 0x5a, 0x4b, 0x62, 0x7c, 0x59, 0x25, 0x49, 0x51, 0xd6, 0x01, 0x31, 0xea, 0xa3, 0xfa, 0x49, 0xad, + 0x29, 0x1d, 0x88, 0xf5, 0x93, 0x63, 0xf6, 0xda, 0xca, 0xa9, 0xb9, 0xac, 0x00, 0x62, 0x47, 0xe6, + 0x7e, 0x3f, 0xff, 0xe7, 0x1c, 0x64, 0x3d, 0x60, 0x76, 0x82, 0xae, 0x46, 0x37, 0xe7, 0x69, 0x74, + 0x3f, 0x82, 0x15, 0x2b, 0x6d, 0xa4, 0xe6, 0x62, 0x48, 0x7d, 0xac, 0x93, 0x02, 0x37, 0x53, 0xfc, + 0xe5, 0x9e, 0x7c, 0xc1, 0x9a, 0x00, 0xc7, 0x58, 0xb7, 0x18, 0x5f, 0x43, 0x99, 0x57, 0xf8, 0x26, + 0x4e, 0x83, 0x93, 0xa9, 0x93, 0x9b, 0xb1, 0x8e, 0x2a, 0x98, 0xfd, 0xc4, 0xa7, 0xc8, 0x7e, 0x22, + 0xdc, 0x5c, 0x62, 0xaa, 0x88, 0x78, 0xfa, 0x8b, 0xbd, 0x36, 0xbc, 0xbc, 0x69, 0xf8, 0xfa, 0xc0, + 0xad, 0xc4, 0x63, 0xd3, 0xad, 0xd4, 0x37, 0x45, 0xee, 0x67, 0xd7, 0x95, 0x2c, 0x17, 0x68, 0x50, + 0x76, 0x85, 0x24, 0x69, 0xf7, 0x7f, 0x38, 0x98, 0xab, 0xb4, 0xb1, 0x6a, 0xd2, 0xb5, 0x2d, 0x7a, + 0xfe, 0xc5, 0x02, 0x6d, 0x44, 0xfc, 0xe7, 0x05, 0x59, 0x18, 0xbf, 0x39, 0xf2, 0xff, 0x32, 0x84, + 0x19, 0x74, 0xe6, 0xfa, 0xf7, 0x10, 0x4f, 0x27, 0xe3, 0xad, 0x00, 0x65, 0x88, 0x9f, 0xe3, 0xef, + 0x8d, 0xc1, 0x72, 0xe6, 0xf9, 0x10, 0x92, 0xe4, 0x31, 0x3d, 0x5a, 0x71, 0x1e, 0xf4, 0xbb, 0xde, + 0xda, 0xf3, 0xab, 0x3e, 0xa8, 0x4d, 0xb7, 0xfb, 0x8f, 0xf3, 0x00, 0xc3, 0x5c, 0x13, 0x3d, 0x85, + 0x05, 0xf7, 0x7b, 0x5e, 0xb4, 0x3e, 0xe2, 0x35, 0x39, 0xbf, 0x11, 0x3e, 0xe8, 0xc8, 0xf4, 0x14, + 0x16, 0xdc, 0x6f, 0xb8, 0x86, 0xcc, 0x42, 0xde, 0x91, 0x0d, 0x99, 0x85, 0x3e, 0xfb, 0x9a, 0x41, + 0x5d, 0xb8, 0x19, 0xf1, 0x8a, 0x07, 0xbd, 0x3d, 0xd9, 0x5b, 0x27, 0xfe, 0x7b, 0x13, 0x3e, 0x07, + 0x12, 0x66, 0x90, 0x0e, 0xb7, 0x22, 0x1f, 0xaf, 0xa0, 0x9d, 0x49, 0x9f, 0xd6, 0xf0, 0xef, 0x4c, + 0x80, 0xe9, 0xcc, 0x39, 0x00, 0x3e, 0xba, 0x63, 0x8e, 0xde, 0x99, 0xf8, 0x29, 0x07, 0x7f, 0x7f, + 0xf2, 0x06, 0xbc, 0x30, 0x83, 0x0e, 0x21, 0xed, 0x6a, 0x9d, 0x22, 0x3e, 0xb4, 0x9f, 0x4a, 0x19, + 0xaf, 0x8f, 0xe8, 0xb5, 0x52, 0x4e, 0xae, 0x6e, 0xd6, 0x90, 0x53, 0xb0, 0x2f, 0x37, 0xe4, 0x14, + 0xd2, 0xfe, 0xf2, 0x6f, 0xbf, 0xef, 0x92, 0x0f, 0xdb, 0xfe, 0xf0, 0x28, 0x21, 0x6c, 0xfb, 0x23, + 0x22, 0x06, 0x61, 0x06, 0x7d, 0x06, 0x4b, 0xde, 0x32, 0x35, 0xda, 0x1c, 0x59, 0x6e, 0xe7, 0x6f, + 0x47, 0x0d, 0xbb, 0x59, 0x7a, 0xab, 0xa2, 0x43, 0x96, 0xa1, 0xd5, 0xd9, 0x21, 0xcb, 0x88, 0x62, + 0xea, 0x8c, 0xe5, 0x9f, 0x3c, 0xb5, 0xbe, 0xa1, 0x7f, 0x0a, 0x2b, 0x51, 0x0e, 0xfd, 0x53, 0x68, + 0x81, 0x50, 0x98, 0x41, 0x0a, 0xac, 0x85, 0x97, 0x9a, 0xd0, 0xbd, 0x89, 0x2a, 0x69, 0xfc, 0xdb, + 0xe3, 0xd0, 0x9c, 0xa9, 0x5a, 0x90, 0x0d, 0xe9, 0x6c, 0x23, 0x61, 0x64, 0xdb, 0x9b, 0x4e, 0x72, + 0x77, 0x82, 0xd6, 0xb8, 0x60, 0x45, 0x1b, 0xbb, 0xff, 0x96, 0x84, 0x04, 0xb9, 0xf6, 0x9b, 0x70, + 0xc3, 0x57, 0x4f, 0x40, 0xb7, 0x47, 0x57, 0x59, 0xf8, 0xad, 0xc8, 0x71, 0x67, 0x0d, 0x2f, 0x61, + 0x39, 0x50, 0x21, 0x40, 0xdb, 0x6e, 0xba, 0xb0, 0x2a, 0x05, 0x7f, 0x67, 0x04, 0x86, 0x9f, 0xb7, + 0xd7, 0xb7, 0x6d, 0x8f, 0x4b, 0x61, 0xbd, 0xbc, 0xa3, 0xfc, 0xd9, 0x17, 0x34, 0xca, 0xf2, 0x7b, + 0x32, 0xc1, 0x2b, 0x57, 0xa8, 0x0f, 0xbb, 0x3b, 0x12, 0xc7, 0x99, 0xe1, 0x73, 0x27, 0xbc, 0x73, + 0x65, 0x50, 0xc8, 0x23, 0x5c, 0x68, 0xa6, 0xc7, 0x0b, 0xa3, 0x50, 0x1c, 0xf6, 0xcf, 0x21, 0xe3, + 0xbf, 0xe7, 0xd1, 0xd6, 0x98, 0xb0, 0x83, 0xdf, 0x8e, 0x46, 0xf0, 0xef, 0x8c, 0xdf, 0xc9, 0xf8, + 0xa5, 0x0a, 0x73, 0x2f, 0x77, 0x47, 0xe2, 0xb8, 0xdd, 0xa2, 0x2b, 0xc2, 0x1d, 0xba, 0xc5, 0x60, + 0x34, 0x3c, 0x74, 0x8b, 0x21, 0x21, 0xb1, 0x30, 0xb3, 0xf7, 0x18, 0x40, 0xee, 0xf6, 0x5f, 0xc9, + 0x12, 0x56, 0x07, 0x3d, 0xb4, 0x11, 0xe8, 0x40, 0x95, 0xd5, 0x41, 0xaf, 0xde, 0xb7, 0x32, 0x2f, + 0x23, 0xf7, 0xf3, 0x39, 0x92, 0x6f, 0xcd, 0x13, 0x02, 0x6b, 0x60, 0xaf, 0x0a, 0x99, 0x21, 0xb5, + 0x44, 0x42, 0x28, 0x74, 0x27, 0x94, 0x07, 0xe9, 0xe7, 0xfb, 0x18, 0x2d, 0x39, 0x8c, 0xc8, 0xe8, + 0xde, 0x27, 0x00, 0x2d, 0x43, 0x91, 0x68, 0x0c, 0x87, 0x36, 0x03, 0x7c, 0x9e, 0x28, 0xb8, 0xdb, + 0xb6, 0x79, 0xfc, 0x05, 0x13, 0xa6, 0x65, 0x28, 0x34, 0xd2, 0xdb, 0xfb, 0x11, 0xa4, 0xa9, 0x30, + 0x67, 0x16, 0xde, 0x38, 0x7a, 0x26, 0x03, 0x5d, 0x3d, 0x19, 0xd9, 0x2b, 0xc3, 0x22, 0x65, 0xc0, + 0xb2, 0x46, 0xb4, 0x15, 0x60, 0x71, 0x44, 0x47, 0x7c, 0x4c, 0x16, 0x08, 0x19, 0x1b, 0xdb, 0x2b, + 0xc2, 0x82, 0xcd, 0xc6, 0x7c, 0xa5, 0xb5, 0xd1, 0xed, 0x10, 0x2e, 0xd6, 0x80, 0x8f, 0x49, 0x9a, + 0x31, 0xb1, 0x86, 0x86, 0xa2, 0xd8, 0xff, 0x67, 0x1a, 0x14, 0x85, 0x65, 0x76, 0xa1, 0xa2, 0xb0, + 0xb1, 0x62, 0xf2, 0x65, 0xbc, 0x65, 0x28, 0xa7, 0x29, 0x42, 0xf4, 0x83, 0xff, 0x0b, 0x00, 0x00, + 0xff, 0xff, 0x2e, 0xde, 0xed, 0x1d, 0x14, 0x3d, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index bcc6f5451..f7a7697ca 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -27,6 +27,7 @@ import ( "net" "strings" + "google.golang.org/grpc/channelz" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal" @@ -192,7 +193,7 @@ type BuildOptions struct { // server can ignore this field. Authority string // ChannelzParentID is the parent ClientConn's channelz ID. - ChannelzParentID int64 + ChannelzParentID *channelz.Identifier // CustomUserAgent is the custom user agent set on the parent ClientConn. // The balancer should set the same custom user agent if it creates a // ClientConn. diff --git a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go index f4ea61746..b1c23eaae 100644 --- a/vendor/google.golang.org/grpc/balancer_conn_wrappers.go +++ b/vendor/google.golang.org/grpc/balancer_conn_wrappers.go @@ -20,130 +20,178 @@ package grpc import ( "fmt" + "strings" "sync" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/internal/balancer/gracefulswitch" "google.golang.org/grpc/internal/buffer" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" "google.golang.org/grpc/resolver" ) -// scStateUpdate contains the subConn and the new state it changed to. +// ccBalancerWrapper sits between the ClientConn and the Balancer. +// +// ccBalancerWrapper implements methods corresponding to the ones on the +// balancer.Balancer interface. The ClientConn is free to call these methods +// concurrently and the ccBalancerWrapper ensures that calls from the ClientConn +// to the Balancer happen synchronously and in order. +// +// ccBalancerWrapper also implements the balancer.ClientConn interface and is +// passed to the Balancer implementations. It invokes unexported methods on the +// ClientConn to handle these calls from the Balancer. +// +// It uses the gracefulswitch.Balancer internally to ensure that balancer +// switches happen in a graceful manner. +type ccBalancerWrapper struct { + cc *ClientConn + + // Since these fields are accessed only from handleXxx() methods which are + // synchronized by the watcher goroutine, we do not need a mutex to protect + // these fields. + balancer *gracefulswitch.Balancer + curBalancerName string + + updateCh *buffer.Unbounded // Updates written on this channel are processed by watcher(). + resultCh *buffer.Unbounded // Results of calls to UpdateClientConnState() are pushed here. + closed *grpcsync.Event // Indicates if close has been called. + done *grpcsync.Event // Indicates if close has completed its work. +} + +// newCCBalancerWrapper creates a new balancer wrapper. The underlying balancer +// is not created until the switchTo() method is invoked. +func newCCBalancerWrapper(cc *ClientConn, bopts balancer.BuildOptions) *ccBalancerWrapper { + ccb := &ccBalancerWrapper{ + cc: cc, + updateCh: buffer.NewUnbounded(), + resultCh: buffer.NewUnbounded(), + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + } + go ccb.watcher() + ccb.balancer = gracefulswitch.NewBalancer(ccb, bopts) + return ccb +} + +// The following xxxUpdate structs wrap the arguments received as part of the +// corresponding update. The watcher goroutine uses the 'type' of the update to +// invoke the appropriate handler routine to handle the update. + +type ccStateUpdate struct { + ccs *balancer.ClientConnState +} + type scStateUpdate struct { sc balancer.SubConn state connectivity.State err error } -// exitIdle contains no data and is just a signal sent on the updateCh in -// ccBalancerWrapper to instruct the balancer to exit idle. -type exitIdle struct{} +type exitIdleUpdate struct{} -// ccBalancerWrapper is a wrapper on top of cc for balancers. -// It implements balancer.ClientConn interface. -type ccBalancerWrapper struct { - cc *ClientConn - balancerMu sync.Mutex // synchronizes calls to the balancer - balancer balancer.Balancer - hasExitIdle bool - updateCh *buffer.Unbounded - closed *grpcsync.Event - done *grpcsync.Event - - mu sync.Mutex - subConns map[*acBalancerWrapper]struct{} +type resolverErrorUpdate struct { + err error } -func newCCBalancerWrapper(cc *ClientConn, b balancer.Builder, bopts balancer.BuildOptions) *ccBalancerWrapper { - ccb := &ccBalancerWrapper{ - cc: cc, - updateCh: buffer.NewUnbounded(), - closed: grpcsync.NewEvent(), - done: grpcsync.NewEvent(), - subConns: make(map[*acBalancerWrapper]struct{}), - } - go ccb.watcher() - ccb.balancer = b.Build(ccb, bopts) - _, ccb.hasExitIdle = ccb.balancer.(balancer.ExitIdler) - return ccb +type switchToUpdate struct { + name string } -// watcher balancer functions sequentially, so the balancer can be implemented -// lock-free. +type subConnUpdate struct { + acbw *acBalancerWrapper +} + +// watcher is a long-running goroutine which reads updates from a channel and +// invokes corresponding methods on the underlying balancer. It ensures that +// these methods are invoked in a synchronous fashion. It also ensures that +// these methods are invoked in the order in which the updates were received. func (ccb *ccBalancerWrapper) watcher() { for { select { - case t := <-ccb.updateCh.Get(): + case u := <-ccb.updateCh.Get(): ccb.updateCh.Load() if ccb.closed.HasFired() { break } - switch u := t.(type) { + switch update := u.(type) { + case *ccStateUpdate: + ccb.handleClientConnStateChange(update.ccs) case *scStateUpdate: - ccb.balancerMu.Lock() - ccb.balancer.UpdateSubConnState(u.sc, balancer.SubConnState{ConnectivityState: u.state, ConnectionError: u.err}) - ccb.balancerMu.Unlock() - case *acBalancerWrapper: - ccb.mu.Lock() - if ccb.subConns != nil { - delete(ccb.subConns, u) - ccb.cc.removeAddrConn(u.getAddrConn(), errConnDrain) - } - ccb.mu.Unlock() - case exitIdle: - if ccb.cc.GetState() == connectivity.Idle { - if ei, ok := ccb.balancer.(balancer.ExitIdler); ok { - // We already checked that the balancer implements - // ExitIdle before pushing the event to updateCh, but - // check conditionally again as defensive programming. - ccb.balancerMu.Lock() - ei.ExitIdle() - ccb.balancerMu.Unlock() - } - } + ccb.handleSubConnStateChange(update) + case *exitIdleUpdate: + ccb.handleExitIdle() + case *resolverErrorUpdate: + ccb.handleResolverError(update.err) + case *switchToUpdate: + ccb.handleSwitchTo(update.name) + case *subConnUpdate: + ccb.handleRemoveSubConn(update.acbw) default: - logger.Errorf("ccBalancerWrapper.watcher: unknown update %+v, type %T", t, t) + logger.Errorf("ccBalancerWrapper.watcher: unknown update %+v, type %T", update, update) } case <-ccb.closed.Done(): } if ccb.closed.HasFired() { - ccb.balancerMu.Lock() - ccb.balancer.Close() - ccb.balancerMu.Unlock() - ccb.mu.Lock() - scs := ccb.subConns - ccb.subConns = nil - ccb.mu.Unlock() - ccb.UpdateState(balancer.State{ConnectivityState: connectivity.Connecting, Picker: nil}) - ccb.done.Fire() - // Fire done before removing the addr conns. We can safely unblock - // ccb.close and allow the removeAddrConns to happen - // asynchronously. - for acbw := range scs { - ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) - } + ccb.handleClose() return } } } -func (ccb *ccBalancerWrapper) close() { - ccb.closed.Fire() - <-ccb.done.Done() -} +// updateClientConnState is invoked by grpc to push a ClientConnState update to +// the underlying balancer. +// +// Unlike other methods invoked by grpc to push updates to the underlying +// balancer, this method cannot simply push the update onto the update channel +// and return. It needs to return the error returned by the underlying balancer +// back to grpc which propagates that to the resolver. +func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error { + ccb.updateCh.Put(&ccStateUpdate{ccs: ccs}) -func (ccb *ccBalancerWrapper) exitIdle() bool { - if !ccb.hasExitIdle { - return false + var res interface{} + select { + case res = <-ccb.resultCh.Get(): + ccb.resultCh.Load() + case <-ccb.closed.Done(): + // Return early if the balancer wrapper is closed while we are waiting for + // the underlying balancer to process a ClientConnState update. + return nil } - ccb.updateCh.Put(exitIdle{}) - return true + // If the returned error is nil, attempting to type assert to error leads to + // panic. So, this needs to handled separately. + if res == nil { + return nil + } + return res.(error) } -func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { +// handleClientConnStateChange handles a ClientConnState update from the update +// channel and invokes the appropriate method on the underlying balancer. +// +// If the addresses specified in the update contain addresses of type "grpclb" +// and the selected LB policy is not "grpclb", these addresses will be filtered +// out and ccs will be modified with the updated address list. +func (ccb *ccBalancerWrapper) handleClientConnStateChange(ccs *balancer.ClientConnState) { + if ccb.curBalancerName != grpclbName { + // Filter any grpclb addresses since we don't have the grpclb balancer. + var addrs []resolver.Address + for _, addr := range ccs.ResolverState.Addresses { + if addr.Type == resolver.GRPCLB { + continue + } + addrs = append(addrs, addr) + } + ccs.ResolverState.Addresses = addrs + } + ccb.resultCh.Put(ccb.balancer.UpdateClientConnState(*ccs)) +} + +// updateSubConnState is invoked by grpc to push a subConn state update to the +// underlying balancer. +func (ccb *ccBalancerWrapper) updateSubConnState(sc balancer.SubConn, s connectivity.State, err error) { // When updating addresses for a SubConn, if the address in use is not in // the new addresses, the old ac will be tearDown() and a new ac will be // created. tearDown() generates a state change with Shutdown state, we @@ -161,44 +209,125 @@ func (ccb *ccBalancerWrapper) handleSubConnStateChange(sc balancer.SubConn, s co }) } -func (ccb *ccBalancerWrapper) updateClientConnState(ccs *balancer.ClientConnState) error { - ccb.balancerMu.Lock() - defer ccb.balancerMu.Unlock() - return ccb.balancer.UpdateClientConnState(*ccs) +// handleSubConnStateChange handles a SubConnState update from the update +// channel and invokes the appropriate method on the underlying balancer. +func (ccb *ccBalancerWrapper) handleSubConnStateChange(update *scStateUpdate) { + ccb.balancer.UpdateSubConnState(update.sc, balancer.SubConnState{ConnectivityState: update.state, ConnectionError: update.err}) +} + +func (ccb *ccBalancerWrapper) exitIdle() { + ccb.updateCh.Put(&exitIdleUpdate{}) +} + +func (ccb *ccBalancerWrapper) handleExitIdle() { + if ccb.cc.GetState() != connectivity.Idle { + return + } + ccb.balancer.ExitIdle() } func (ccb *ccBalancerWrapper) resolverError(err error) { - ccb.balancerMu.Lock() - defer ccb.balancerMu.Unlock() + ccb.updateCh.Put(&resolverErrorUpdate{err: err}) +} + +func (ccb *ccBalancerWrapper) handleResolverError(err error) { ccb.balancer.ResolverError(err) } +// switchTo is invoked by grpc to instruct the balancer wrapper to switch to the +// LB policy identified by name. +// +// ClientConn calls newCCBalancerWrapper() at creation time. Upon receipt of the +// first good update from the name resolver, it determines the LB policy to use +// and invokes the switchTo() method. Upon receipt of every subsequent update +// from the name resolver, it invokes this method. +// +// the ccBalancerWrapper keeps track of the current LB policy name, and skips +// the graceful balancer switching process if the name does not change. +func (ccb *ccBalancerWrapper) switchTo(name string) { + ccb.updateCh.Put(&switchToUpdate{name: name}) +} + +// handleSwitchTo handles a balancer switch update from the update channel. It +// calls the SwitchTo() method on the gracefulswitch.Balancer with a +// balancer.Builder corresponding to name. If no balancer.Builder is registered +// for the given name, it uses the default LB policy which is "pick_first". +func (ccb *ccBalancerWrapper) handleSwitchTo(name string) { + // TODO: Other languages use case-insensitive balancer registries. We should + // switch as well. See: https://github.com/grpc/grpc-go/issues/5288. + if strings.EqualFold(ccb.curBalancerName, name) { + return + } + + // TODO: Ensure that name is a registered LB policy when we get here. + // We currently only validate the `loadBalancingConfig` field. We need to do + // the same for the `loadBalancingPolicy` field and reject the service config + // if the specified policy is not registered. + builder := balancer.Get(name) + if builder == nil { + channelz.Warningf(logger, ccb.cc.channelzID, "Channel switches to new LB policy %q, since the specified LB policy %q was not registered", PickFirstBalancerName, name) + builder = newPickfirstBuilder() + } else { + channelz.Infof(logger, ccb.cc.channelzID, "Channel switches to new LB policy %q", name) + } + + if err := ccb.balancer.SwitchTo(builder); err != nil { + channelz.Errorf(logger, ccb.cc.channelzID, "Channel failed to build new LB policy %q: %v", name, err) + return + } + ccb.curBalancerName = builder.Name() +} + +// handleRemoveSucConn handles a request from the underlying balancer to remove +// a subConn. +// +// See comments in RemoveSubConn() for more details. +func (ccb *ccBalancerWrapper) handleRemoveSubConn(acbw *acBalancerWrapper) { + ccb.cc.removeAddrConn(acbw.getAddrConn(), errConnDrain) +} + +func (ccb *ccBalancerWrapper) close() { + ccb.closed.Fire() + <-ccb.done.Done() +} + +func (ccb *ccBalancerWrapper) handleClose() { + ccb.balancer.Close() + ccb.done.Fire() +} + func (ccb *ccBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { if len(addrs) <= 0 { return nil, fmt.Errorf("grpc: cannot create SubConn with empty address list") } - ccb.mu.Lock() - defer ccb.mu.Unlock() - if ccb.subConns == nil { - return nil, fmt.Errorf("grpc: ClientConn balancer wrapper was closed") - } ac, err := ccb.cc.newAddrConn(addrs, opts) if err != nil { + channelz.Warningf(logger, ccb.cc.channelzID, "acBalancerWrapper: NewSubConn: failed to newAddrConn: %v", err) return nil, err } acbw := &acBalancerWrapper{ac: ac} acbw.ac.mu.Lock() ac.acbw = acbw acbw.ac.mu.Unlock() - ccb.subConns[acbw] = struct{}{} return acbw, nil } func (ccb *ccBalancerWrapper) RemoveSubConn(sc balancer.SubConn) { - // The RemoveSubConn() is handled in the run() goroutine, to avoid deadlock - // during switchBalancer() if the old balancer calls RemoveSubConn() in its - // Close(). - ccb.updateCh.Put(sc) + // Before we switched the ccBalancerWrapper to use gracefulswitch.Balancer, it + // was required to handle the RemoveSubConn() method asynchronously by pushing + // the update onto the update channel. This was done to avoid a deadlock as + // switchBalancer() was holding cc.mu when calling Close() on the old + // balancer, which would in turn call RemoveSubConn(). + // + // With the use of gracefulswitch.Balancer in ccBalancerWrapper, handling this + // asynchronously is probably not required anymore since the switchTo() method + // handles the balancer switch by pushing the update onto the channel. + // TODO(easwars): Handle this inline. + acbw, ok := sc.(*acBalancerWrapper) + if !ok { + return + } + ccb.updateCh.Put(&subConnUpdate{acbw: acbw}) } func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { @@ -210,11 +339,6 @@ func (ccb *ccBalancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resol } func (ccb *ccBalancerWrapper) UpdateState(s balancer.State) { - ccb.mu.Lock() - defer ccb.mu.Unlock() - if ccb.subConns == nil { - return - } // Update picker before updating state. Even though the ordering here does // not matter, it can lead to multiple calls of Pick in the common start-up // case where we wait for ready and then perform an RPC. If the picker is diff --git a/vendor/google.golang.org/grpc/channelz/channelz.go b/vendor/google.golang.org/grpc/channelz/channelz.go new file mode 100644 index 000000000..a220c47c5 --- /dev/null +++ b/vendor/google.golang.org/grpc/channelz/channelz.go @@ -0,0 +1,36 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package channelz exports internals of the channelz implementation as required +// by other gRPC packages. +// +// The implementation of the channelz spec as defined in +// https://github.com/grpc/proposal/blob/master/A14-channelz.md, is provided by +// the `internal/channelz` package. +// +// Experimental +// +// Notice: All APIs in this package are experimental and may be removed in a +// later release. +package channelz + +import "google.golang.org/grpc/internal/channelz" + +// Identifier is an opaque identifier which uniquely identifies an entity in the +// channelz database. +type Identifier = channelz.Identifier diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index f9af78913..3ed6eb8e7 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -159,23 +159,20 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * } }() - if channelz.IsOn() { - if cc.dopts.channelzParentID != 0 { - cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, cc.dopts.channelzParentID, target) - channelz.AddTraceEvent(logger, cc.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Channel Created", - Severity: channelz.CtInfo, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested Channel(id:%d) created", cc.channelzID), - Severity: channelz.CtInfo, - }, - }) - } else { - cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, 0, target) - channelz.Info(logger, cc.channelzID, "Channel Created") - } - cc.csMgr.channelzID = cc.channelzID + pid := cc.dopts.channelzParentID + cc.channelzID = channelz.RegisterChannel(&channelzChannel{cc}, pid, target) + ted := &channelz.TraceEventDesc{ + Desc: "Channel created", + Severity: channelz.CtInfo, } + if cc.dopts.channelzParentID != nil { + ted.Parent = &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Nested Channel(id:%d) created", cc.channelzID.Int()), + Severity: channelz.CtInfo, + } + } + channelz.AddTraceEvent(logger, cc.channelzID, 1, ted) + cc.csMgr.channelzID = cc.channelzID if cc.dopts.copts.TransportCredentials == nil && cc.dopts.copts.CredsBundle == nil { return nil, errNoTransportSecurity @@ -281,7 +278,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * if creds := cc.dopts.copts.TransportCredentials; creds != nil { credsClone = creds.Clone() } - cc.balancerBuildOpts = balancer.BuildOptions{ + cc.balancerWrapper = newCCBalancerWrapper(cc, balancer.BuildOptions{ DialCreds: credsClone, CredsBundle: cc.dopts.copts.CredsBundle, Dialer: cc.dopts.copts.Dialer, @@ -289,7 +286,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * CustomUserAgent: cc.dopts.copts.UserAgent, ChannelzParentID: cc.channelzID, Target: cc.parsedTarget, - } + }) // Build the resolver. rWrapper, err := newCCResolverWrapper(cc, resolverBuilder) @@ -398,7 +395,7 @@ type connectivityStateManager struct { mu sync.Mutex state connectivity.State notifyChan chan struct{} - channelzID int64 + channelzID *channelz.Identifier } // updateState updates the connectivity.State of ClientConn. @@ -464,34 +461,36 @@ var _ ClientConnInterface = (*ClientConn)(nil) // handshakes. It also handles errors on established connections by // re-resolving the name and reconnecting. type ClientConn struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context // Initialized using the background context at dial time. + cancel context.CancelFunc // Cancelled on close. - target string - parsedTarget resolver.Target - authority string - dopts dialOptions - csMgr *connectivityStateManager - - balancerBuildOpts balancer.BuildOptions - blockingpicker *pickerWrapper + // The following are initialized at dial time, and are read-only after that. + target string // User's dial target. + parsedTarget resolver.Target // See parseTargetAndFindResolver(). + authority string // See determineAuthority(). + dopts dialOptions // Default and user specified dial options. + channelzID *channelz.Identifier // Channelz identifier for the channel. + balancerWrapper *ccBalancerWrapper // Uses gracefulswitch.balancer underneath. + // The following provide their own synchronization, and therefore don't + // require cc.mu to be held to access them. + csMgr *connectivityStateManager + blockingpicker *pickerWrapper safeConfigSelector iresolver.SafeConfigSelector + czData *channelzData + retryThrottler atomic.Value // Updated from service config. - mu sync.RWMutex - resolverWrapper *ccResolverWrapper - sc *ServiceConfig - conns map[*addrConn]struct{} - // Keepalive parameter can be updated if a GoAway is received. - mkp keepalive.ClientParameters - curBalancerName string - balancerWrapper *ccBalancerWrapper - retryThrottler atomic.Value - + // firstResolveEvent is used to track whether the name resolver sent us at + // least one update. RPCs block on this event. firstResolveEvent *grpcsync.Event - channelzID int64 // channelz unique identification number - czData *channelzData + // mu protects the following fields. + // TODO: split mu so the same mutex isn't used for everything. + mu sync.RWMutex + resolverWrapper *ccResolverWrapper // Initialized in Dial; cleared in Close. + sc *ServiceConfig // Latest service config received from the resolver. + conns map[*addrConn]struct{} // Set to nil on close. + mkp keepalive.ClientParameters // May be updated upon receipt of a GoAway. lceMu sync.Mutex // protects lastConnectionError lastConnectionError error @@ -536,14 +535,7 @@ func (cc *ClientConn) GetState() connectivity.State { // Notice: This API is EXPERIMENTAL and may be changed or removed in a later // release. func (cc *ClientConn) Connect() { - cc.mu.Lock() - defer cc.mu.Unlock() - if cc.balancerWrapper != nil && cc.balancerWrapper.exitIdle() { - return - } - for ac := range cc.conns { - go ac.connect() - } + cc.balancerWrapper.exitIdle() } func (cc *ClientConn) scWatcher() { @@ -623,9 +615,7 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { // with the new addresses. cc.maybeApplyDefaultServiceConfig(nil) - if cc.balancerWrapper != nil { - cc.balancerWrapper.resolverError(err) - } + cc.balancerWrapper.resolverError(err) // No addresses are valid with err set; return early. cc.mu.Unlock() @@ -653,16 +643,10 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { cc.applyServiceConfigAndBalancer(sc, configSelector, s.Addresses) } else { ret = balancer.ErrBadResolverState - if cc.balancerWrapper == nil { - var err error - if s.ServiceConfig.Err != nil { - err = status.Errorf(codes.Unavailable, "error parsing service config: %v", s.ServiceConfig.Err) - } else { - err = status.Errorf(codes.Unavailable, "illegal service config type: %T", s.ServiceConfig.Config) - } - cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{cc.sc}) - cc.blockingpicker.updatePicker(base.NewErrPicker(err)) - cc.csMgr.updateState(connectivity.TransientFailure) + if cc.sc == nil { + // Apply the failing LB only if we haven't received valid service config + // from the name resolver in the past. + cc.applyFailingLB(s.ServiceConfig) cc.mu.Unlock() return ret } @@ -670,24 +654,12 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { } var balCfg serviceconfig.LoadBalancingConfig - if cc.dopts.balancerBuilder == nil && cc.sc != nil && cc.sc.lbConfig != nil { + if cc.sc != nil && cc.sc.lbConfig != nil { balCfg = cc.sc.lbConfig.cfg } - - cbn := cc.curBalancerName bw := cc.balancerWrapper cc.mu.Unlock() - if cbn != grpclbName { - // Filter any grpclb addresses since we don't have the grpclb balancer. - for i := 0; i < len(s.Addresses); { - if s.Addresses[i].Type == resolver.GRPCLB { - copy(s.Addresses[i:], s.Addresses[i+1:]) - s.Addresses = s.Addresses[:len(s.Addresses)-1] - continue - } - i++ - } - } + uccsErr := bw.updateClientConnState(&balancer.ClientConnState{ResolverState: s, BalancerConfig: balCfg}) if ret == nil { ret = uccsErr // prefer ErrBadResolver state since any other error is @@ -696,56 +668,28 @@ func (cc *ClientConn) updateResolverState(s resolver.State, err error) error { return ret } -// switchBalancer starts the switching from current balancer to the balancer -// with the given name. -// -// It will NOT send the current address list to the new balancer. If needed, -// caller of this function should send address list to the new balancer after -// this function returns. +// applyFailingLB is akin to configuring an LB policy on the channel which +// always fails RPCs. Here, an actual LB policy is not configured, but an always +// erroring picker is configured, which returns errors with information about +// what was invalid in the received service config. A config selector with no +// service config is configured, and the connectivity state of the channel is +// set to TransientFailure. // // Caller must hold cc.mu. -func (cc *ClientConn) switchBalancer(name string) { - if strings.EqualFold(cc.curBalancerName, name) { - return - } - - channelz.Infof(logger, cc.channelzID, "ClientConn switching balancer to %q", name) - if cc.dopts.balancerBuilder != nil { - channelz.Info(logger, cc.channelzID, "ignoring balancer switching: Balancer DialOption used instead") - return - } - if cc.balancerWrapper != nil { - // Don't hold cc.mu while closing the balancers. The balancers may call - // methods that require cc.mu (e.g. cc.NewSubConn()). Holding the mutex - // would cause a deadlock in that case. - cc.mu.Unlock() - cc.balancerWrapper.close() - cc.mu.Lock() - } - - builder := balancer.Get(name) - if builder == nil { - channelz.Warningf(logger, cc.channelzID, "Channel switches to new LB policy %q due to fallback from invalid balancer name", PickFirstBalancerName) - channelz.Infof(logger, cc.channelzID, "failed to get balancer builder for: %v, using pick_first instead", name) - builder = newPickfirstBuilder() +func (cc *ClientConn) applyFailingLB(sc *serviceconfig.ParseResult) { + var err error + if sc.Err != nil { + err = status.Errorf(codes.Unavailable, "error parsing service config: %v", sc.Err) } else { - channelz.Infof(logger, cc.channelzID, "Channel switches to new LB policy %q", name) + err = status.Errorf(codes.Unavailable, "illegal service config type: %T", sc.Config) } - - cc.curBalancerName = builder.Name() - cc.balancerWrapper = newCCBalancerWrapper(cc, builder, cc.balancerBuildOpts) + cc.safeConfigSelector.UpdateConfigSelector(&defaultConfigSelector{nil}) + cc.blockingpicker.updatePicker(base.NewErrPicker(err)) + cc.csMgr.updateState(connectivity.TransientFailure) } func (cc *ClientConn) handleSubConnStateChange(sc balancer.SubConn, s connectivity.State, err error) { - cc.mu.Lock() - if cc.conns == nil { - cc.mu.Unlock() - return - } - // TODO(bar switching) send updates to all balancer wrappers when balancer - // gracefully switching is supported. - cc.balancerWrapper.handleSubConnStateChange(sc, s, err) - cc.mu.Unlock() + cc.balancerWrapper.updateSubConnState(sc, s, err) } // newAddrConn creates an addrConn for addrs and adds it to cc.conns. @@ -768,17 +712,21 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub cc.mu.Unlock() return nil, ErrClientConnClosing } - if channelz.IsOn() { - ac.channelzID = channelz.RegisterSubChannel(ac, cc.channelzID, "") - channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Subchannel Created", - Severity: channelz.CtInfo, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID), - Severity: channelz.CtInfo, - }, - }) + + var err error + ac.channelzID, err = channelz.RegisterSubChannel(ac, cc.channelzID, "") + if err != nil { + return nil, err } + channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ + Desc: "Subchannel created", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel(id:%d) created", ac.channelzID.Int()), + Severity: channelz.CtInfo, + }, + }) + cc.conns[ac] = struct{}{} cc.mu.Unlock() return ac, nil @@ -991,35 +939,26 @@ func (cc *ClientConn) applyServiceConfigAndBalancer(sc *ServiceConfig, configSel cc.retryThrottler.Store((*retryThrottler)(nil)) } - if cc.dopts.balancerBuilder == nil { - // Only look at balancer types and switch balancer if balancer dial - // option is not set. - var newBalancerName string - if cc.sc != nil && cc.sc.lbConfig != nil { - newBalancerName = cc.sc.lbConfig.name - } else { - var isGRPCLB bool - for _, a := range addrs { - if a.Type == resolver.GRPCLB { - isGRPCLB = true - break - } - } - if isGRPCLB { - newBalancerName = grpclbName - } else if cc.sc != nil && cc.sc.LB != nil { - newBalancerName = *cc.sc.LB - } else { - newBalancerName = PickFirstBalancerName + var newBalancerName string + if cc.sc != nil && cc.sc.lbConfig != nil { + newBalancerName = cc.sc.lbConfig.name + } else { + var isGRPCLB bool + for _, a := range addrs { + if a.Type == resolver.GRPCLB { + isGRPCLB = true + break } } - cc.switchBalancer(newBalancerName) - } else if cc.balancerWrapper == nil { - // Balancer dial option was set, and this is the first time handling - // resolved addresses. Build a balancer with dopts.balancerBuilder. - cc.curBalancerName = cc.dopts.balancerBuilder.Name() - cc.balancerWrapper = newCCBalancerWrapper(cc, cc.dopts.balancerBuilder, cc.balancerBuildOpts) + if isGRPCLB { + newBalancerName = grpclbName + } else if cc.sc != nil && cc.sc.LB != nil { + newBalancerName = *cc.sc.LB + } else { + newBalancerName = PickFirstBalancerName + } } + cc.balancerWrapper.switchTo(newBalancerName) } func (cc *ClientConn) resolveNow(o resolver.ResolveNowOptions) { @@ -1070,11 +1009,11 @@ func (cc *ClientConn) Close() error { rWrapper := cc.resolverWrapper cc.resolverWrapper = nil bWrapper := cc.balancerWrapper - cc.balancerWrapper = nil cc.mu.Unlock() + // The order of closing matters here since the balancer wrapper assumes the + // picker is closed before it is closed. cc.blockingpicker.close() - if bWrapper != nil { bWrapper.close() } @@ -1085,22 +1024,22 @@ func (cc *ClientConn) Close() error { for ac := range conns { ac.tearDown(ErrClientConnClosing) } - if channelz.IsOn() { - ted := &channelz.TraceEventDesc{ - Desc: "Channel Deleted", + ted := &channelz.TraceEventDesc{ + Desc: "Channel deleted", + Severity: channelz.CtInfo, + } + if cc.dopts.channelzParentID != nil { + ted.Parent = &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Nested channel(id:%d) deleted", cc.channelzID.Int()), Severity: channelz.CtInfo, } - if cc.dopts.channelzParentID != 0 { - ted.Parent = &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Nested channel(id:%d) deleted", cc.channelzID), - Severity: channelz.CtInfo, - } - } - channelz.AddTraceEvent(logger, cc.channelzID, 0, ted) - // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to - // the entity being deleted, and thus prevent it from being deleted right away. - channelz.RemoveEntry(cc.channelzID) } + channelz.AddTraceEvent(logger, cc.channelzID, 0, ted) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from being + // deleted right away. + channelz.RemoveEntry(cc.channelzID) + return nil } @@ -1130,7 +1069,7 @@ type addrConn struct { backoffIdx int // Needs to be stateful for resetConnectBackoff. resetBackoff chan struct{} - channelzID int64 // channelz unique identification number. + channelzID *channelz.Identifier czData *channelzData } @@ -1312,14 +1251,12 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne connectCtx, cancel := context.WithDeadline(ac.ctx, connectDeadline) defer cancel() - if channelz.IsOn() { - copts.ChannelzParentID = ac.channelzID - } + copts.ChannelzParentID = ac.channelzID newTr, err := transport.NewClientTransport(connectCtx, ac.cc.ctx, addr, copts, func() { prefaceReceived.Fire() }, onGoAway, onClose) if err != nil { // newTr is either nil, or closed. - channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v. Err: %v", addr, err) + channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %s. Err: %v", addr, err) return err } @@ -1332,7 +1269,7 @@ func (ac *addrConn) createTransport(addr resolver.Address, copts transport.Conne newTr.Close(transport.ErrConnClosing) if connectCtx.Err() == context.DeadlineExceeded { err := errors.New("failed to receive server preface within timeout") - channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %v: %v", addr, err) + channelz.Warningf(logger, ac.channelzID, "grpc: addrConn.createTransport failed to connect to %s: %v", addr, err) return err } return nil @@ -1497,19 +1434,18 @@ func (ac *addrConn) tearDown(err error) { curTr.GracefulClose() ac.mu.Lock() } - if channelz.IsOn() { - channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ - Desc: "Subchannel Deleted", + channelz.AddTraceEvent(logger, ac.channelzID, 0, &channelz.TraceEventDesc{ + Desc: "Subchannel deleted", + Severity: channelz.CtInfo, + Parent: &channelz.TraceEventDesc{ + Desc: fmt.Sprintf("Subchannel(id:%d) deleted", ac.channelzID.Int()), Severity: channelz.CtInfo, - Parent: &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Subchanel(id:%d) deleted", ac.channelzID), - Severity: channelz.CtInfo, - }, - }) - // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add trace reference to - // the entity being deleted, and thus prevent it from being deleted right away. - channelz.RemoveEntry(ac.channelzID) - } + }, + }) + // TraceEvent needs to be called before RemoveEntry, as TraceEvent may add + // trace reference to the entity being deleted, and thus prevent it from + // being deleted right away. + channelz.RemoveEntry(ac.channelzID) ac.mu.Unlock() } diff --git a/vendor/google.golang.org/grpc/dialoptions.go b/vendor/google.golang.org/grpc/dialoptions.go index c4bf09f9e..f2f605a17 100644 --- a/vendor/google.golang.org/grpc/dialoptions.go +++ b/vendor/google.golang.org/grpc/dialoptions.go @@ -20,12 +20,11 @@ package grpc import ( "context" - "fmt" "net" "time" "google.golang.org/grpc/backoff" - "google.golang.org/grpc/balancer" + "google.golang.org/grpc/channelz" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/internal" @@ -45,19 +44,17 @@ type dialOptions struct { chainUnaryInts []UnaryClientInterceptor chainStreamInts []StreamClientInterceptor - cp Compressor - dc Decompressor - bs internalbackoff.Strategy - block bool - returnLastError bool - timeout time.Duration - scChan <-chan ServiceConfig - authority string - copts transport.ConnectOptions - callOptions []CallOption - // This is used by WithBalancerName dial option. - balancerBuilder balancer.Builder - channelzParentID int64 + cp Compressor + dc Decompressor + bs internalbackoff.Strategy + block bool + returnLastError bool + timeout time.Duration + scChan <-chan ServiceConfig + authority string + copts transport.ConnectOptions + callOptions []CallOption + channelzParentID *channelz.Identifier disableServiceConfig bool disableRetry bool disableHealthCheck bool @@ -195,25 +192,6 @@ func WithDecompressor(dc Decompressor) DialOption { }) } -// WithBalancerName sets the balancer that the ClientConn will be initialized -// with. Balancer registered with balancerName will be used. This function -// panics if no balancer was registered by balancerName. -// -// The balancer cannot be overridden by balancer option specified by service -// config. -// -// Deprecated: use WithDefaultServiceConfig and WithDisableServiceConfig -// instead. Will be removed in a future 1.x release. -func WithBalancerName(balancerName string) DialOption { - builder := balancer.Get(balancerName) - if builder == nil { - panic(fmt.Sprintf("grpc.WithBalancerName: no balancer is registered for name %v", balancerName)) - } - return newFuncDialOption(func(o *dialOptions) { - o.balancerBuilder = builder - }) -} - // WithServiceConfig returns a DialOption which has a channel to read the // service configuration. // @@ -304,8 +282,8 @@ func WithReturnConnectionError() DialOption { // WithCredentialsBundle or WithPerRPCCredentials) which require transport // security is incompatible and will cause grpc.Dial() to fail. // -// Deprecated: use WithTransportCredentials and insecure.NewCredentials() instead. -// Will be supported throughout 1.x. +// Deprecated: use WithTransportCredentials and insecure.NewCredentials() +// instead. Will be supported throughout 1.x. func WithInsecure() DialOption { return newFuncDialOption(func(o *dialOptions) { o.copts.TransportCredentials = insecure.NewCredentials() @@ -498,7 +476,7 @@ func WithAuthority(a string) DialOption { // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. -func WithChannelzParentID(id int64) DialOption { +func WithChannelzParentID(id *channelz.Identifier) DialOption { return newFuncDialOption(func(o *dialOptions) { o.channelzParentID = id }) diff --git a/vendor/google.golang.org/grpc/internal/balancer/gracefulswitch/gracefulswitch.go b/vendor/google.golang.org/grpc/internal/balancer/gracefulswitch/gracefulswitch.go new file mode 100644 index 000000000..7ba8f4d18 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/balancer/gracefulswitch/gracefulswitch.go @@ -0,0 +1,382 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package gracefulswitch implements a graceful switch load balancer. +package gracefulswitch + +import ( + "errors" + "fmt" + "sync" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +var errBalancerClosed = errors.New("gracefulSwitchBalancer is closed") +var _ balancer.Balancer = (*Balancer)(nil) + +// NewBalancer returns a graceful switch Balancer. +func NewBalancer(cc balancer.ClientConn, opts balancer.BuildOptions) *Balancer { + return &Balancer{ + cc: cc, + bOpts: opts, + } +} + +// Balancer is a utility to gracefully switch from one balancer to +// a new balancer. It implements the balancer.Balancer interface. +type Balancer struct { + bOpts balancer.BuildOptions + cc balancer.ClientConn + + // mu protects the following fields and all fields within balancerCurrent + // and balancerPending. mu does not need to be held when calling into the + // child balancers, as all calls into these children happen only as a direct + // result of a call into the gracefulSwitchBalancer, which are also + // guaranteed to be synchronous. There is one exception: an UpdateState call + // from a child balancer when current and pending are populated can lead to + // calling Close() on the current. To prevent that racing with an + // UpdateSubConnState from the channel, we hold currentMu during Close and + // UpdateSubConnState calls. + mu sync.Mutex + balancerCurrent *balancerWrapper + balancerPending *balancerWrapper + closed bool // set to true when this balancer is closed + + // currentMu must be locked before mu. This mutex guards against this + // sequence of events: UpdateSubConnState() called, finds the + // balancerCurrent, gives up lock, updateState comes in, causes Close() on + // balancerCurrent before the UpdateSubConnState is called on the + // balancerCurrent. + currentMu sync.Mutex +} + +// swap swaps out the current lb with the pending lb and updates the ClientConn. +// The caller must hold gsb.mu. +func (gsb *Balancer) swap() { + gsb.cc.UpdateState(gsb.balancerPending.lastState) + cur := gsb.balancerCurrent + gsb.balancerCurrent = gsb.balancerPending + gsb.balancerPending = nil + go func() { + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + cur.Close() + }() +} + +// Helper function that checks if the balancer passed in is current or pending. +// The caller must hold gsb.mu. +func (gsb *Balancer) balancerCurrentOrPending(bw *balancerWrapper) bool { + return bw == gsb.balancerCurrent || bw == gsb.balancerPending +} + +// SwitchTo initializes the graceful switch process, which completes based on +// connectivity state changes on the current/pending balancer. Thus, the switch +// process is not complete when this method returns. This method must be called +// synchronously alongside the rest of the balancer.Balancer methods this +// Graceful Switch Balancer implements. +func (gsb *Balancer) SwitchTo(builder balancer.Builder) error { + gsb.mu.Lock() + if gsb.closed { + gsb.mu.Unlock() + return errBalancerClosed + } + bw := &balancerWrapper{ + gsb: gsb, + lastState: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + subconns: make(map[balancer.SubConn]bool), + } + balToClose := gsb.balancerPending // nil if there is no pending balancer + if gsb.balancerCurrent == nil { + gsb.balancerCurrent = bw + } else { + gsb.balancerPending = bw + } + gsb.mu.Unlock() + balToClose.Close() + // This function takes a builder instead of a balancer because builder.Build + // can call back inline, and this utility needs to handle the callbacks. + newBalancer := builder.Build(bw, gsb.bOpts) + if newBalancer == nil { + // This is illegal and should never happen; we clear the balancerWrapper + // we were constructing if it happens to avoid a potential panic. + gsb.mu.Lock() + if gsb.balancerPending != nil { + gsb.balancerPending = nil + } else { + gsb.balancerCurrent = nil + } + gsb.mu.Unlock() + return balancer.ErrBadResolverState + } + + // This write doesn't need to take gsb.mu because this field never gets read + // or written to on any calls from the current or pending. Calls from grpc + // to this balancer are guaranteed to be called synchronously, so this + // bw.Balancer field will never be forwarded to until this SwitchTo() + // function returns. + bw.Balancer = newBalancer + return nil +} + +// Returns nil if the graceful switch balancer is closed. +func (gsb *Balancer) latestBalancer() *balancerWrapper { + gsb.mu.Lock() + defer gsb.mu.Unlock() + if gsb.balancerPending != nil { + return gsb.balancerPending + } + return gsb.balancerCurrent +} + +// UpdateClientConnState forwards the update to the latest balancer created. +func (gsb *Balancer) UpdateClientConnState(state balancer.ClientConnState) error { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return errBalancerClosed + } + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + return balToUpdate.UpdateClientConnState(state) +} + +// ResolverError forwards the error to the latest balancer created. +func (gsb *Balancer) ResolverError(err error) { + // The resolver data is only relevant to the most recent LB Policy. + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return + } + // Perform this call without gsb.mu to prevent deadlocks if the child calls + // back into the channel. The latest balancer can never be closed during a + // call from the channel, even without gsb.mu held. + balToUpdate.ResolverError(err) +} + +// ExitIdle forwards the call to the latest balancer created. +// +// If the latest balancer does not support ExitIdle, the subConns are +// re-connected to manually. +func (gsb *Balancer) ExitIdle() { + balToUpdate := gsb.latestBalancer() + if balToUpdate == nil { + return + } + // There is no need to protect this read with a mutex, as the write to the + // Balancer field happens in SwitchTo, which completes before this can be + // called. + if ei, ok := balToUpdate.Balancer.(balancer.ExitIdler); ok { + ei.ExitIdle() + return + } + for sc := range balToUpdate.subconns { + sc.Connect() + } +} + +// UpdateSubConnState forwards the update to the appropriate child. +func (gsb *Balancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + gsb.currentMu.Lock() + defer gsb.currentMu.Unlock() + gsb.mu.Lock() + // Forward update to the appropriate child. Even if there is a pending + // balancer, the current balancer should continue to get SubConn updates to + // maintain the proper state while the pending is still connecting. + var balToUpdate *balancerWrapper + if gsb.balancerCurrent != nil && gsb.balancerCurrent.subconns[sc] { + balToUpdate = gsb.balancerCurrent + } else if gsb.balancerPending != nil && gsb.balancerPending.subconns[sc] { + balToUpdate = gsb.balancerPending + } + gsb.mu.Unlock() + if balToUpdate == nil { + // SubConn belonged to a stale lb policy that has not yet fully closed, + // or the balancer was already closed. + return + } + balToUpdate.UpdateSubConnState(sc, state) +} + +// Close closes any active child balancers. +func (gsb *Balancer) Close() { + gsb.mu.Lock() + gsb.closed = true + currentBalancerToClose := gsb.balancerCurrent + gsb.balancerCurrent = nil + pendingBalancerToClose := gsb.balancerPending + gsb.balancerPending = nil + gsb.mu.Unlock() + + currentBalancerToClose.Close() + pendingBalancerToClose.Close() +} + +// balancerWrapper wraps a balancer.Balancer, and overrides some Balancer +// methods to help cleanup SubConns created by the wrapped balancer. +// +// It implements the balancer.ClientConn interface and is passed down in that +// capacity to the wrapped balancer. It maintains a set of subConns created by +// the wrapped balancer and calls from the latter to create/update/remove +// SubConns update this set before being forwarded to the parent ClientConn. +// State updates from the wrapped balancer can result in invocation of the +// graceful switch logic. +type balancerWrapper struct { + balancer.Balancer + gsb *Balancer + + lastState balancer.State + subconns map[balancer.SubConn]bool // subconns created by this balancer +} + +func (bw *balancerWrapper) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + if state.ConnectivityState == connectivity.Shutdown { + bw.gsb.mu.Lock() + delete(bw.subconns, sc) + bw.gsb.mu.Unlock() + } + // There is no need to protect this read with a mutex, as the write to the + // Balancer field happens in SwitchTo, which completes before this can be + // called. + bw.Balancer.UpdateSubConnState(sc, state) +} + +// Close closes the underlying LB policy and removes the subconns it created. bw +// must not be referenced via balancerCurrent or balancerPending in gsb when +// called. gsb.mu must not be held. Does not panic with a nil receiver. +func (bw *balancerWrapper) Close() { + // before Close is called. + if bw == nil { + return + } + // There is no need to protect this read with a mutex, as Close() is + // impossible to be called concurrently with the write in SwitchTo(). The + // callsites of Close() for this balancer in Graceful Switch Balancer will + // never be called until SwitchTo() returns. + bw.Balancer.Close() + bw.gsb.mu.Lock() + for sc := range bw.subconns { + bw.gsb.cc.RemoveSubConn(sc) + } + bw.gsb.mu.Unlock() +} + +func (bw *balancerWrapper) UpdateState(state balancer.State) { + // Hold the mutex for this entire call to ensure it cannot occur + // concurrently with other updateState() calls. This causes updates to + // lastState and calls to cc.UpdateState to happen atomically. + bw.gsb.mu.Lock() + defer bw.gsb.mu.Unlock() + bw.lastState = state + + if !bw.gsb.balancerCurrentOrPending(bw) { + return + } + + if bw == bw.gsb.balancerCurrent { + // In the case that the current balancer exits READY, and there is a pending + // balancer, you can forward the pending balancer's cached State up to + // ClientConn and swap the pending into the current. This is because there + // is no reason to gracefully switch from and keep using the old policy as + // the ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Ready && bw.gsb.balancerPending != nil { + bw.gsb.swap() + return + } + // Even if there is a pending balancer waiting to be gracefully switched to, + // continue to forward current balancer updates to the Client Conn. Ignoring + // state + picker from the current would cause undefined behavior/cause the + // system to behave incorrectly from the current LB policies perspective. + // Also, the current LB is still being used by grpc to choose SubConns per + // RPC, and thus should use the most updated form of the current balancer. + bw.gsb.cc.UpdateState(state) + return + } + // This method is now dealing with a state update from the pending balancer. + // If the current balancer is currently in a state other than READY, the new + // policy can be swapped into place immediately. This is because there is no + // reason to gracefully switch from and keep using the old policy as the + // ClientConn is not connected to any backends. + if state.ConnectivityState != connectivity.Connecting || bw.gsb.balancerCurrent.lastState.ConnectivityState != connectivity.Ready { + bw.gsb.swap() + } +} + +func (bw *balancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.gsb.mu.Unlock() + + sc, err := bw.gsb.cc.NewSubConn(addrs, opts) + if err != nil { + return nil, err + } + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { // balancer was closed during this call + bw.gsb.cc.RemoveSubConn(sc) + bw.gsb.mu.Unlock() + return nil, fmt.Errorf("%T at address %p that called NewSubConn is deleted", bw, bw) + } + bw.subconns[sc] = true + bw.gsb.mu.Unlock() + return sc, nil +} + +func (bw *balancerWrapper) ResolveNow(opts resolver.ResolveNowOptions) { + // Ignore ResolveNow requests from anything other than the most recent + // balancer, because older balancers were already removed from the config. + if bw != bw.gsb.latestBalancer() { + return + } + bw.gsb.cc.ResolveNow(opts) +} + +func (bw *balancerWrapper) RemoveSubConn(sc balancer.SubConn) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return + } + bw.gsb.mu.Unlock() + bw.gsb.cc.RemoveSubConn(sc) +} + +func (bw *balancerWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + bw.gsb.mu.Lock() + if !bw.gsb.balancerCurrentOrPending(bw) { + bw.gsb.mu.Unlock() + return + } + bw.gsb.mu.Unlock() + bw.gsb.cc.UpdateAddresses(sc, addrs) +} + +func (bw *balancerWrapper) Target() string { + return bw.gsb.cc.Target() +} diff --git a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go index 5cc3aeddb..0a25ce43f 100644 --- a/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/binarylog.go @@ -31,7 +31,7 @@ import ( // Logger is the global binary logger. It can be used to get binary logger for // each method. type Logger interface { - getMethodLogger(methodName string) *MethodLogger + GetMethodLogger(methodName string) MethodLogger } // binLogger is the global binary logger for the binary. One of this should be @@ -49,17 +49,24 @@ func SetLogger(l Logger) { binLogger = l } +// GetLogger gets the binarg logger. +// +// Only call this at init time. +func GetLogger() Logger { + return binLogger +} + // GetMethodLogger returns the methodLogger for the given methodName. // // methodName should be in the format of "/service/method". // // Each methodLogger returned by this method is a new instance. This is to // generate sequence id within the call. -func GetMethodLogger(methodName string) *MethodLogger { +func GetMethodLogger(methodName string) MethodLogger { if binLogger == nil { return nil } - return binLogger.getMethodLogger(methodName) + return binLogger.GetMethodLogger(methodName) } func init() { @@ -68,17 +75,29 @@ func init() { binLogger = NewLoggerFromConfigString(configStr) } -type methodLoggerConfig struct { +// MethodLoggerConfig contains the setting for logging behavior of a method +// logger. Currently, it contains the max length of header and message. +type MethodLoggerConfig struct { // Max length of header and message. - hdr, msg uint64 + Header, Message uint64 +} + +// LoggerConfig contains the config for loggers to create method loggers. +type LoggerConfig struct { + All *MethodLoggerConfig + Services map[string]*MethodLoggerConfig + Methods map[string]*MethodLoggerConfig + + Blacklist map[string]struct{} } type logger struct { - all *methodLoggerConfig - services map[string]*methodLoggerConfig - methods map[string]*methodLoggerConfig + config LoggerConfig +} - blacklist map[string]struct{} +// NewLoggerFromConfig builds a logger with the given LoggerConfig. +func NewLoggerFromConfig(config LoggerConfig) Logger { + return &logger{config: config} } // newEmptyLogger creates an empty logger. The map fields need to be filled in @@ -88,57 +107,57 @@ func newEmptyLogger() *logger { } // Set method logger for "*". -func (l *logger) setDefaultMethodLogger(ml *methodLoggerConfig) error { - if l.all != nil { +func (l *logger) setDefaultMethodLogger(ml *MethodLoggerConfig) error { + if l.config.All != nil { return fmt.Errorf("conflicting global rules found") } - l.all = ml + l.config.All = ml return nil } // Set method logger for "service/*". // // New methodLogger with same service overrides the old one. -func (l *logger) setServiceMethodLogger(service string, ml *methodLoggerConfig) error { - if _, ok := l.services[service]; ok { +func (l *logger) setServiceMethodLogger(service string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Services[service]; ok { return fmt.Errorf("conflicting service rules for service %v found", service) } - if l.services == nil { - l.services = make(map[string]*methodLoggerConfig) + if l.config.Services == nil { + l.config.Services = make(map[string]*MethodLoggerConfig) } - l.services[service] = ml + l.config.Services[service] = ml return nil } // Set method logger for "service/method". // // New methodLogger with same method overrides the old one. -func (l *logger) setMethodMethodLogger(method string, ml *methodLoggerConfig) error { - if _, ok := l.blacklist[method]; ok { +func (l *logger) setMethodMethodLogger(method string, ml *MethodLoggerConfig) error { + if _, ok := l.config.Blacklist[method]; ok { return fmt.Errorf("conflicting blacklist rules for method %v found", method) } - if _, ok := l.methods[method]; ok { + if _, ok := l.config.Methods[method]; ok { return fmt.Errorf("conflicting method rules for method %v found", method) } - if l.methods == nil { - l.methods = make(map[string]*methodLoggerConfig) + if l.config.Methods == nil { + l.config.Methods = make(map[string]*MethodLoggerConfig) } - l.methods[method] = ml + l.config.Methods[method] = ml return nil } // Set blacklist method for "-service/method". func (l *logger) setBlacklist(method string) error { - if _, ok := l.blacklist[method]; ok { + if _, ok := l.config.Blacklist[method]; ok { return fmt.Errorf("conflicting blacklist rules for method %v found", method) } - if _, ok := l.methods[method]; ok { + if _, ok := l.config.Methods[method]; ok { return fmt.Errorf("conflicting method rules for method %v found", method) } - if l.blacklist == nil { - l.blacklist = make(map[string]struct{}) + if l.config.Blacklist == nil { + l.config.Blacklist = make(map[string]struct{}) } - l.blacklist[method] = struct{}{} + l.config.Blacklist[method] = struct{}{} return nil } @@ -148,23 +167,23 @@ func (l *logger) setBlacklist(method string) error { // // Each methodLogger returned by this method is a new instance. This is to // generate sequence id within the call. -func (l *logger) getMethodLogger(methodName string) *MethodLogger { +func (l *logger) GetMethodLogger(methodName string) MethodLogger { s, m, err := grpcutil.ParseMethod(methodName) if err != nil { grpclogLogger.Infof("binarylogging: failed to parse %q: %v", methodName, err) return nil } - if ml, ok := l.methods[s+"/"+m]; ok { - return newMethodLogger(ml.hdr, ml.msg) + if ml, ok := l.config.Methods[s+"/"+m]; ok { + return newMethodLogger(ml.Header, ml.Message) } - if _, ok := l.blacklist[s+"/"+m]; ok { + if _, ok := l.config.Blacklist[s+"/"+m]; ok { return nil } - if ml, ok := l.services[s]; ok { - return newMethodLogger(ml.hdr, ml.msg) + if ml, ok := l.config.Services[s]; ok { + return newMethodLogger(ml.Header, ml.Message) } - if l.all == nil { + if l.config.All == nil { return nil } - return newMethodLogger(l.all.hdr, l.all.msg) + return newMethodLogger(l.config.All.Header, l.config.All.Message) } diff --git a/vendor/google.golang.org/grpc/internal/binarylog/env_config.go b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go index d8f4e7602..ab589a76b 100644 --- a/vendor/google.golang.org/grpc/internal/binarylog/env_config.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/env_config.go @@ -89,7 +89,7 @@ func (l *logger) fillMethodLoggerWithConfigString(config string) error { if err != nil { return fmt.Errorf("invalid config: %q, %v", config, err) } - if err := l.setDefaultMethodLogger(&methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setDefaultMethodLogger(&MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } return nil @@ -104,11 +104,11 @@ func (l *logger) fillMethodLoggerWithConfigString(config string) error { return fmt.Errorf("invalid header/message length config: %q, %v", suffix, err) } if m == "*" { - if err := l.setServiceMethodLogger(s, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setServiceMethodLogger(s, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } } else { - if err := l.setMethodMethodLogger(s+"/"+m, &methodLoggerConfig{hdr: hdr, msg: msg}); err != nil { + if err := l.setMethodMethodLogger(s+"/"+m, &MethodLoggerConfig{Header: hdr, Message: msg}); err != nil { return fmt.Errorf("invalid config: %v", err) } } diff --git a/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go index 0cdb41831..24df0a1a0 100644 --- a/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go +++ b/vendor/google.golang.org/grpc/internal/binarylog/method_logger.go @@ -48,7 +48,11 @@ func (g *callIDGenerator) reset() { var idGen callIDGenerator // MethodLogger is the sub-logger for each method. -type MethodLogger struct { +type MethodLogger interface { + Log(LogEntryConfig) +} + +type methodLogger struct { headerMaxLen, messageMaxLen uint64 callID uint64 @@ -57,8 +61,8 @@ type MethodLogger struct { sink Sink // TODO(blog): make this plugable. } -func newMethodLogger(h, m uint64) *MethodLogger { - return &MethodLogger{ +func newMethodLogger(h, m uint64) *methodLogger { + return &methodLogger{ headerMaxLen: h, messageMaxLen: m, @@ -69,8 +73,10 @@ func newMethodLogger(h, m uint64) *MethodLogger { } } -// Log creates a proto binary log entry, and logs it to the sink. -func (ml *MethodLogger) Log(c LogEntryConfig) { +// Build is an internal only method for building the proto message out of the +// input event. It's made public to enable other library to reuse as much logic +// in methodLogger as possible. +func (ml *methodLogger) Build(c LogEntryConfig) *pb.GrpcLogEntry { m := c.toProto() timestamp, _ := ptypes.TimestampProto(time.Now()) m.Timestamp = timestamp @@ -85,11 +91,15 @@ func (ml *MethodLogger) Log(c LogEntryConfig) { case *pb.GrpcLogEntry_Message: m.PayloadTruncated = ml.truncateMessage(pay.Message) } - - ml.sink.Write(m) + return m } -func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { +// Log creates a proto binary log entry, and logs it to the sink. +func (ml *methodLogger) Log(c LogEntryConfig) { + ml.sink.Write(ml.Build(c)) +} + +func (ml *methodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { if ml.headerMaxLen == maxUInt { return false } @@ -119,7 +129,7 @@ func (ml *MethodLogger) truncateMetadata(mdPb *pb.Metadata) (truncated bool) { return truncated } -func (ml *MethodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { +func (ml *methodLogger) truncateMessage(msgPb *pb.Message) (truncated bool) { if ml.messageMaxLen == maxUInt { return false } diff --git a/vendor/google.golang.org/grpc/internal/channelz/funcs.go b/vendor/google.golang.org/grpc/internal/channelz/funcs.go index ea660a147..777cbcd79 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/funcs.go +++ b/vendor/google.golang.org/grpc/internal/channelz/funcs.go @@ -25,6 +25,7 @@ package channelz import ( "context" + "errors" "fmt" "sort" "sync" @@ -184,54 +185,77 @@ func GetServer(id int64) *ServerMetric { return db.get().GetServer(id) } -// RegisterChannel registers the given channel c in channelz database with ref -// as its reference name, and add it to the child list of its parent (identified -// by pid). pid = 0 means no parent. It returns the unique channelz tracking id -// assigned to this channel. -func RegisterChannel(c Channel, pid int64, ref string) int64 { +// RegisterChannel registers the given channel c in the channelz database with +// ref as its reference name, and adds it to the child list of its parent +// (identified by pid). pid == nil means no parent. +// +// Returns a unique channelz identifier assigned to this channel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterChannel(c Channel, pid *Identifier, ref string) *Identifier { id := idGen.genID() + var parent int64 + isTopChannel := true + if pid != nil { + isTopChannel = false + parent = pid.Int() + } + + if !IsOn() { + return newIdentifer(RefChannel, id, pid) + } + cn := &channel{ refName: ref, c: c, subChans: make(map[int64]string), nestedChans: make(map[int64]string), id: id, - pid: pid, + pid: parent, trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } - if pid == 0 { - db.get().addChannel(id, cn, true, pid) - } else { - db.get().addChannel(id, cn, false, pid) - } - return id + db.get().addChannel(id, cn, isTopChannel, parent) + return newIdentifer(RefChannel, id, pid) } -// RegisterSubChannel registers the given channel c in channelz database with ref -// as its reference name, and add it to the child list of its parent (identified -// by pid). It returns the unique channelz tracking id assigned to this subchannel. -func RegisterSubChannel(c Channel, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a SubChannel's parent id cannot be 0") - return 0 +// RegisterSubChannel registers the given subChannel c in the channelz database +// with ref as its reference name, and adds it to the child list of its parent +// (identified by pid). +// +// Returns a unique channelz identifier assigned to this subChannel. +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterSubChannel(c Channel, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a SubChannel's parent id cannot be nil") } id := idGen.genID() + if !IsOn() { + return newIdentifer(RefSubChannel, id, pid), nil + } + sc := &subChannel{ refName: ref, c: c, sockets: make(map[int64]string), id: id, - pid: pid, + pid: pid.Int(), trace: &channelTrace{createdTime: time.Now(), events: make([]*TraceEvent, 0, getMaxTraceEntry())}, } - db.get().addSubChannel(id, sc, pid) - return id + db.get().addSubChannel(id, sc, pid.Int()) + return newIdentifer(RefSubChannel, id, pid), nil } // RegisterServer registers the given server s in channelz database. It returns // the unique channelz tracking id assigned to this server. -func RegisterServer(s Server, ref string) int64 { +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterServer(s Server, ref string) *Identifier { id := idGen.genID() + if !IsOn() { + return newIdentifer(RefServer, id, nil) + } + svr := &server{ refName: ref, s: s, @@ -240,71 +264,92 @@ func RegisterServer(s Server, ref string) int64 { id: id, } db.get().addServer(id, svr) - return id + return newIdentifer(RefServer, id, nil) } // RegisterListenSocket registers the given listen socket s in channelz database // with ref as its reference name, and add it to the child list of its parent // (identified by pid). It returns the unique channelz tracking id assigned to // this listen socket. -func RegisterListenSocket(s Socket, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a ListenSocket's parent id cannot be 0") - return 0 +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterListenSocket(s Socket, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a ListenSocket's parent id cannot be 0") } id := idGen.genID() - ls := &listenSocket{refName: ref, s: s, id: id, pid: pid} - db.get().addListenSocket(id, ls, pid) - return id + if !IsOn() { + return newIdentifer(RefListenSocket, id, pid), nil + } + + ls := &listenSocket{refName: ref, s: s, id: id, pid: pid.Int()} + db.get().addListenSocket(id, ls, pid.Int()) + return newIdentifer(RefListenSocket, id, pid), nil } // RegisterNormalSocket registers the given normal socket s in channelz database -// with ref as its reference name, and add it to the child list of its parent +// with ref as its reference name, and adds it to the child list of its parent // (identified by pid). It returns the unique channelz tracking id assigned to // this normal socket. -func RegisterNormalSocket(s Socket, pid int64, ref string) int64 { - if pid == 0 { - logger.Error("a NormalSocket's parent id cannot be 0") - return 0 +// +// If channelz is not turned ON, the channelz database is not mutated. +func RegisterNormalSocket(s Socket, pid *Identifier, ref string) (*Identifier, error) { + if pid == nil { + return nil, errors.New("a NormalSocket's parent id cannot be 0") } id := idGen.genID() - ns := &normalSocket{refName: ref, s: s, id: id, pid: pid} - db.get().addNormalSocket(id, ns, pid) - return id + if !IsOn() { + return newIdentifer(RefNormalSocket, id, pid), nil + } + + ns := &normalSocket{refName: ref, s: s, id: id, pid: pid.Int()} + db.get().addNormalSocket(id, ns, pid.Int()) + return newIdentifer(RefNormalSocket, id, pid), nil } // RemoveEntry removes an entry with unique channelz tracking id to be id from // channelz database. -func RemoveEntry(id int64) { - db.get().removeEntry(id) +// +// If channelz is not turned ON, this function is a no-op. +func RemoveEntry(id *Identifier) { + if !IsOn() { + return + } + db.get().removeEntry(id.Int()) } -// TraceEventDesc is what the caller of AddTraceEvent should provide to describe the event to be added -// to the channel trace. -// The Parent field is optional. It is used for event that will be recorded in the entity's parent -// trace also. +// TraceEventDesc is what the caller of AddTraceEvent should provide to describe +// the event to be added to the channel trace. +// +// The Parent field is optional. It is used for an event that will be recorded +// in the entity's parent trace. type TraceEventDesc struct { Desc string Severity Severity Parent *TraceEventDesc } -// AddTraceEvent adds trace related to the entity with specified id, using the provided TraceEventDesc. -func AddTraceEvent(l grpclog.DepthLoggerV2, id int64, depth int, desc *TraceEventDesc) { - for d := desc; d != nil; d = d.Parent { - switch d.Severity { - case CtUnknown, CtInfo: - l.InfoDepth(depth+1, d.Desc) - case CtWarning: - l.WarningDepth(depth+1, d.Desc) - case CtError: - l.ErrorDepth(depth+1, d.Desc) - } +// AddTraceEvent adds trace related to the entity with specified id, using the +// provided TraceEventDesc. +// +// If channelz is not turned ON, this will simply log the event descriptions. +func AddTraceEvent(l grpclog.DepthLoggerV2, id *Identifier, depth int, desc *TraceEventDesc) { + // Log only the trace description associated with the bottom most entity. + switch desc.Severity { + case CtUnknown, CtInfo: + l.InfoDepth(depth+1, withParens(id)+desc.Desc) + case CtWarning: + l.WarningDepth(depth+1, withParens(id)+desc.Desc) + case CtError: + l.ErrorDepth(depth+1, withParens(id)+desc.Desc) } + if getMaxTraceEntry() == 0 { return } - db.get().traceEvent(id, desc) + if IsOn() { + db.get().traceEvent(id.Int(), desc) + } } // channelMap is the storage data structure for channelz. diff --git a/vendor/google.golang.org/grpc/internal/channelz/id.go b/vendor/google.golang.org/grpc/internal/channelz/id.go new file mode 100644 index 000000000..c9a27acd3 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/channelz/id.go @@ -0,0 +1,75 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package channelz + +import "fmt" + +// Identifier is an opaque identifier which uniquely identifies an entity in the +// channelz database. +type Identifier struct { + typ RefChannelType + id int64 + str string + pid *Identifier +} + +// Type returns the entity type corresponding to id. +func (id *Identifier) Type() RefChannelType { + return id.typ +} + +// Int returns the integer identifier corresponding to id. +func (id *Identifier) Int() int64 { + return id.id +} + +// String returns a string representation of the entity corresponding to id. +// +// This includes some information about the parent as well. Examples: +// Top-level channel: [Channel #channel-number] +// Nested channel: [Channel #parent-channel-number Channel #channel-number] +// Sub channel: [Channel #parent-channel SubChannel #subchannel-number] +func (id *Identifier) String() string { + return id.str +} + +// Equal returns true if other is the same as id. +func (id *Identifier) Equal(other *Identifier) bool { + if (id != nil) != (other != nil) { + return false + } + if id == nil && other == nil { + return true + } + return id.typ == other.typ && id.id == other.id && id.pid == other.pid +} + +// NewIdentifierForTesting returns a new opaque identifier to be used only for +// testing purposes. +func NewIdentifierForTesting(typ RefChannelType, id int64, pid *Identifier) *Identifier { + return newIdentifer(typ, id, pid) +} + +func newIdentifer(typ RefChannelType, id int64, pid *Identifier) *Identifier { + str := fmt.Sprintf("%s #%d", typ, id) + if pid != nil { + str = fmt.Sprintf("%s %s", pid, str) + } + return &Identifier{typ: typ, id: id, str: str, pid: pid} +} diff --git a/vendor/google.golang.org/grpc/internal/channelz/logging.go b/vendor/google.golang.org/grpc/internal/channelz/logging.go index b0013f9c8..8e13a3d2c 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/logging.go +++ b/vendor/google.golang.org/grpc/internal/channelz/logging.go @@ -26,77 +26,54 @@ import ( var logger = grpclog.Component("channelz") +func withParens(id *Identifier) string { + return "[" + id.String() + "] " +} + // Info logs and adds a trace event if channelz is on. -func Info(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtInfo, - }) - } else { - l.InfoDepth(1, args...) - } +func Info(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtInfo, + }) } // Infof logs and adds a trace event if channelz is on. -func Infof(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtInfo, - }) - } else { - l.InfoDepth(1, msg) - } +func Infof(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtInfo, + }) } // Warning logs and adds a trace event if channelz is on. -func Warning(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtWarning, - }) - } else { - l.WarningDepth(1, args...) - } +func Warning(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtWarning, + }) } // Warningf logs and adds a trace event if channelz is on. -func Warningf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtWarning, - }) - } else { - l.WarningDepth(1, msg) - } +func Warningf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtWarning, + }) } // Error logs and adds a trace event if channelz is on. -func Error(l grpclog.DepthLoggerV2, id int64, args ...interface{}) { - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: fmt.Sprint(args...), - Severity: CtError, - }) - } else { - l.ErrorDepth(1, args...) - } +func Error(l grpclog.DepthLoggerV2, id *Identifier, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprint(args...), + Severity: CtError, + }) } // Errorf logs and adds a trace event if channelz is on. -func Errorf(l grpclog.DepthLoggerV2, id int64, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - if IsOn() { - AddTraceEvent(l, id, 1, &TraceEventDesc{ - Desc: msg, - Severity: CtError, - }) - } else { - l.ErrorDepth(1, msg) - } +func Errorf(l grpclog.DepthLoggerV2, id *Identifier, format string, args ...interface{}) { + AddTraceEvent(l, id, 1, &TraceEventDesc{ + Desc: fmt.Sprintf(format, args...), + Severity: CtError, + }) } diff --git a/vendor/google.golang.org/grpc/internal/channelz/types.go b/vendor/google.golang.org/grpc/internal/channelz/types.go index 3c595d154..ad0ce4dab 100644 --- a/vendor/google.golang.org/grpc/internal/channelz/types.go +++ b/vendor/google.golang.org/grpc/internal/channelz/types.go @@ -686,12 +686,33 @@ const ( type RefChannelType int const ( + // RefUnknown indicates an unknown entity type, the zero value for this type. + RefUnknown RefChannelType = iota // RefChannel indicates the referenced entity is a Channel. - RefChannel RefChannelType = iota + RefChannel // RefSubChannel indicates the referenced entity is a SubChannel. RefSubChannel + // RefServer indicates the referenced entity is a Server. + RefServer + // RefListenSocket indicates the referenced entity is a ListenSocket. + RefListenSocket + // RefNormalSocket indicates the referenced entity is a NormalSocket. + RefNormalSocket ) +var refChannelTypeToString = map[RefChannelType]string{ + RefUnknown: "Unknown", + RefChannel: "Channel", + RefSubChannel: "SubChannel", + RefServer: "Server", + RefListenSocket: "ListenSocket", + RefNormalSocket: "NormalSocket", +} + +func (r RefChannelType) String() string { + return refChannelTypeToString[r] +} + func (c *channelTrace) dumpData() *ChannelTrace { c.mu.Lock() ct := &ChannelTrace{EventNum: c.eventCount, CreationTime: c.createdTime} diff --git a/vendor/google.golang.org/grpc/internal/internal.go b/vendor/google.golang.org/grpc/internal/internal.go index 20fb880f3..6d355b0b0 100644 --- a/vendor/google.golang.org/grpc/internal/internal.go +++ b/vendor/google.golang.org/grpc/internal/internal.go @@ -85,3 +85,9 @@ const ( // that supports backend returned by grpclb balancer. CredsBundleModeBackendFromBalancer = "backend-from-balancer" ) + +// RLSLoadBalancingPolicyName is the name of the RLS LB policy. +// +// It currently has an experimental suffix which would be removed once +// end-to-end testing of the policy is completed. +const RLSLoadBalancingPolicyName = "rls_experimental" diff --git a/vendor/google.golang.org/grpc/internal/metadata/metadata.go b/vendor/google.golang.org/grpc/internal/metadata/metadata.go index b8733dbf3..b2980f8ac 100644 --- a/vendor/google.golang.org/grpc/internal/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/internal/metadata/metadata.go @@ -22,6 +22,9 @@ package metadata import ( + "fmt" + "strings" + "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" ) @@ -72,3 +75,46 @@ func Set(addr resolver.Address, md metadata.MD) resolver.Address { addr.Attributes = addr.Attributes.WithValue(mdKey, mdValue(md)) return addr } + +// Validate returns an error if the input md contains invalid keys or values. +// +// If the header is not a pseudo-header, the following items are checked: +// - header names must contain one or more characters from this set [0-9 a-z _ - .]. +// - if the header-name ends with a "-bin" suffix, no validation of the header value is performed. +// - otherwise, the header value must contain one or more characters from the set [%x20-%x7E]. +func Validate(md metadata.MD) error { + for k, vals := range md { + // pseudo-header will be ignored + if k[0] == ':' { + continue + } + // check key, for i that saving a conversion if not using for range + for i := 0; i < len(k); i++ { + r := k[i] + if !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9') && r != '.' && r != '-' && r != '_' { + return fmt.Errorf("header key %q contains illegal characters not in [0-9a-z-_.]", k) + } + } + if strings.HasSuffix(k, "-bin") { + continue + } + // check value + for _, val := range vals { + if hasNotPrintable(val) { + return fmt.Errorf("header key %q contains value with non-printable ASCII characters", k) + } + } + } + return nil +} + +// hasNotPrintable return true if msg contains any characters which are not in %x20-%x7E +func hasNotPrintable(msg string) bool { + // for i that saving a conversion if not using for range + for i := 0; i < len(msg); i++ { + if msg[i] < 0x20 || msg[i] > 0x7E { + return true + } + } + return false +} diff --git a/vendor/google.golang.org/grpc/internal/pretty/pretty.go b/vendor/google.golang.org/grpc/internal/pretty/pretty.go new file mode 100644 index 000000000..0177af4b5 --- /dev/null +++ b/vendor/google.golang.org/grpc/internal/pretty/pretty.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pretty defines helper functions to pretty-print structs for logging. +package pretty + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/golang/protobuf/jsonpb" + protov1 "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/encoding/protojson" + protov2 "google.golang.org/protobuf/proto" +) + +const jsonIndent = " " + +// ToJSON marshals the input into a json string. +// +// If marshal fails, it falls back to fmt.Sprintf("%+v"). +func ToJSON(e interface{}) string { + switch ee := e.(type) { + case protov1.Message: + mm := jsonpb.Marshaler{Indent: jsonIndent} + ret, err := mm.MarshalToString(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return ret + case protov2.Message: + mm := protojson.MarshalOptions{ + Multiline: true, + Indent: jsonIndent, + } + ret, err := mm.Marshal(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return string(ret) + default: + ret, err := json.MarshalIndent(ee, "", jsonIndent) + if err != nil { + return fmt.Sprintf("%+v", ee) + } + return string(ret) + } +} + +// FormatJSON formats the input json bytes with indentation. +// +// If Indent fails, it returns the unchanged input as string. +func FormatJSON(b []byte) string { + var out bytes.Buffer + err := json.Indent(&out, b, "", jsonIndent) + if err != nil { + return string(b) + } + return out.String() +} diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_client.go b/vendor/google.golang.org/grpc/internal/transport/http2_client.go index f0c72d337..38ed3d566 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_client.go @@ -132,7 +132,7 @@ type http2Client struct { kpDormant bool // Fields below are for channelz metric collection. - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData onGoAway func(GoAwayReason) @@ -351,8 +351,9 @@ func newHTTP2Client(connectCtx, ctx context.Context, addr resolver.Address, opts } t.statsHandler.HandleConn(t.ctx, connBegin) } - if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) + t.channelzID, err = channelz.RegisterNormalSocket(t, opts.ChannelzParentID, fmt.Sprintf("%s -> %s", t.localAddr, t.remoteAddr)) + if err != nil { + return nil, err } if t.keepaliveEnabled { t.kpDormancyCond = sync.NewCond(&t.mu) @@ -898,9 +899,7 @@ func (t *http2Client) Close(err error) { t.controlBuf.finish() t.cancel() t.conn.Close() - if channelz.IsOn() { - channelz.RemoveEntry(t.channelzID) - } + channelz.RemoveEntry(t.channelzID) // Append info about previous goaways if there were any, since this may be important // for understanding the root cause for this connection to be closed. _, goAwayDebugMessage := t.GetGoAwayReason() diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_server.go b/vendor/google.golang.org/grpc/internal/transport/http2_server.go index 2c6eaf0e5..0956b500c 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_server.go @@ -36,6 +36,7 @@ import ( "golang.org/x/net/http2" "golang.org/x/net/http2/hpack" "google.golang.org/grpc/internal/grpcutil" + "google.golang.org/grpc/internal/syscall" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -117,7 +118,7 @@ type http2Server struct { idle time.Time // Fields below are for channelz metric collection. - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData bufferPool *bufferPool @@ -231,6 +232,11 @@ func NewServerTransport(conn net.Conn, config *ServerConfig) (_ ServerTransport, if kp.Timeout == 0 { kp.Timeout = defaultServerKeepaliveTimeout } + if kp.Time != infinity { + if err = syscall.SetTCPUserTimeout(conn, kp.Timeout); err != nil { + return nil, connectionErrorf(false, err, "transport: failed to set TCP_USER_TIMEOUT: %v", err) + } + } kep := config.KeepalivePolicy if kep.MinTime == 0 { kep.MinTime = defaultKeepalivePolicyMinTime @@ -275,12 +281,12 @@ func NewServerTransport(conn net.Conn, config *ServerConfig) (_ ServerTransport, connBegin := &stats.ConnBegin{} t.stats.HandleConn(t.ctx, connBegin) } - if channelz.IsOn() { - t.channelzID = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) + t.channelzID, err = channelz.RegisterNormalSocket(t, config.ChannelzParentID, fmt.Sprintf("%s -> %s", t.remoteAddr, t.localAddr)) + if err != nil { + return nil, err } t.connectionID = atomic.AddUint64(&serverConnectionCounter, 1) - t.framer.writer.Flush() defer func() { @@ -1210,9 +1216,7 @@ func (t *http2Server) Close() { if err := t.conn.Close(); err != nil && logger.V(logLevel) { logger.Infof("transport: error closing conn during Close: %v", err) } - if channelz.IsOn() { - channelz.RemoveEntry(t.channelzID) - } + channelz.RemoveEntry(t.channelzID) // Cancel all active streams. for _, s := range streams { s.cancel() diff --git a/vendor/google.golang.org/grpc/internal/transport/transport.go b/vendor/google.golang.org/grpc/internal/transport/transport.go index 0c43efaa6..a9ce717f1 100644 --- a/vendor/google.golang.org/grpc/internal/transport/transport.go +++ b/vendor/google.golang.org/grpc/internal/transport/transport.go @@ -34,6 +34,7 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" "google.golang.org/grpc/resolver" @@ -529,7 +530,7 @@ type ServerConfig struct { InitialConnWindowSize int32 WriteBufferSize int ReadBufferSize int - ChannelzParentID int64 + ChannelzParentID *channelz.Identifier MaxHeaderListSize *uint32 HeaderTableSize *uint32 } @@ -563,7 +564,7 @@ type ConnectOptions struct { // ReadBufferSize sets the size of read buffer, which in turn determines how much data can be read at most for one read syscall. ReadBufferSize int // ChannelzParentID sets the addrConn id which initiate the creation of this client transport. - ChannelzParentID int64 + ChannelzParentID *channelz.Identifier // MaxHeaderListSize sets the max (uncompressed) size of header list that is prepared to be received. MaxHeaderListSize *uint32 // UseProxy specifies if a proxy should be used. diff --git a/vendor/google.golang.org/grpc/metadata/metadata.go b/vendor/google.golang.org/grpc/metadata/metadata.go index 3604c7819..8e0f6abe8 100644 --- a/vendor/google.golang.org/grpc/metadata/metadata.go +++ b/vendor/google.golang.org/grpc/metadata/metadata.go @@ -188,7 +188,9 @@ func FromIncomingContext(ctx context.Context) (MD, bool) { // map, and there's no guarantee that the MD attached to the context is // created using our helper functions. key := strings.ToLower(k) - out[key] = v + s := make([]string, len(v)) + copy(s, v) + out[key] = s } return out, true } @@ -226,7 +228,9 @@ func FromOutgoingContext(ctx context.Context) (MD, bool) { // map, and there's no guarantee that the MD attached to the context is // created using our helper functions. key := strings.ToLower(k) - out[key] = v + s := make([]string, len(v)) + copy(s, v) + out[key] = s } for _, added := range raw.added { if len(added)%2 == 1 { diff --git a/vendor/google.golang.org/grpc/pickfirst.go b/vendor/google.golang.org/grpc/pickfirst.go index 5168b62b0..fb7a99e0a 100644 --- a/vendor/google.golang.org/grpc/pickfirst.go +++ b/vendor/google.golang.org/grpc/pickfirst.go @@ -44,79 +44,107 @@ func (*pickfirstBuilder) Name() string { } type pickfirstBalancer struct { - state connectivity.State - cc balancer.ClientConn - sc balancer.SubConn + state connectivity.State + cc balancer.ClientConn + subConn balancer.SubConn } func (b *pickfirstBalancer) ResolverError(err error) { - switch b.state { - case connectivity.TransientFailure, connectivity.Idle, connectivity.Connecting: - // Set a failing picker if we don't have a good picker. - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, - }) - } if logger.V(2) { logger.Infof("pickfirstBalancer: ResolverError called with error %v", err) } + if b.subConn == nil { + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("name resolver error: %v", err)}, + }) } -func (b *pickfirstBalancer) UpdateClientConnState(cs balancer.ClientConnState) error { - if len(cs.ResolverState.Addresses) == 0 { +func (b *pickfirstBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if len(state.ResolverState.Addresses) == 0 { + // The resolver reported an empty address list. Treat it like an error by + // calling b.ResolverError. + if b.subConn != nil { + // Remove the old subConn. All addresses were removed, so it is no longer + // valid. + b.cc.RemoveSubConn(b.subConn) + b.subConn = nil + } b.ResolverError(errors.New("produced zero addresses")) return balancer.ErrBadResolverState } - if b.sc == nil { - var err error - b.sc, err = b.cc.NewSubConn(cs.ResolverState.Addresses, balancer.NewSubConnOptions{}) - if err != nil { - if logger.V(2) { - logger.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) - } - b.state = connectivity.TransientFailure - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.TransientFailure, - Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, - }) - return balancer.ErrBadResolverState - } - b.state = connectivity.Idle - b.cc.UpdateState(balancer.State{ConnectivityState: connectivity.Idle, Picker: &picker{result: balancer.PickResult{SubConn: b.sc}}}) - b.sc.Connect() - } else { - b.cc.UpdateAddresses(b.sc, cs.ResolverState.Addresses) - b.sc.Connect() + + if b.subConn != nil { + b.cc.UpdateAddresses(b.subConn, state.ResolverState.Addresses) + return nil } + + subConn, err := b.cc.NewSubConn(state.ResolverState.Addresses, balancer.NewSubConnOptions{}) + if err != nil { + if logger.V(2) { + logger.Errorf("pickfirstBalancer: failed to NewSubConn: %v", err) + } + b.state = connectivity.TransientFailure + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: &picker{err: fmt.Errorf("error creating connection: %v", err)}, + }) + return balancer.ErrBadResolverState + } + b.subConn = subConn + b.state = connectivity.Idle + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.Idle, + Picker: &picker{result: balancer.PickResult{SubConn: b.subConn}}, + }) + b.subConn.Connect() return nil } -func (b *pickfirstBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { +func (b *pickfirstBalancer) UpdateSubConnState(subConn balancer.SubConn, state balancer.SubConnState) { if logger.V(2) { - logger.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", sc, s) + logger.Infof("pickfirstBalancer: UpdateSubConnState: %p, %v", subConn, state) } - if b.sc != sc { + if b.subConn != subConn { if logger.V(2) { - logger.Infof("pickfirstBalancer: ignored state change because sc is not recognized") + logger.Infof("pickfirstBalancer: ignored state change because subConn is not recognized") } return } - b.state = s.ConnectivityState - if s.ConnectivityState == connectivity.Shutdown { - b.sc = nil + b.state = state.ConnectivityState + if state.ConnectivityState == connectivity.Shutdown { + b.subConn = nil return } - switch s.ConnectivityState { + switch state.ConnectivityState { case connectivity.Ready: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{result: balancer.PickResult{SubConn: sc}}}) + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{result: balancer.PickResult{SubConn: subConn}}, + }) case connectivity.Connecting: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &picker{err: balancer.ErrNoSubConnAvailable}}) + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: balancer.ErrNoSubConnAvailable}, + }) case connectivity.Idle: - b.cc.UpdateState(balancer.State{ConnectivityState: s.ConnectivityState, Picker: &idlePicker{sc: sc}}) + b.cc.UpdateState(balancer.State{ + ConnectivityState: state.ConnectivityState, + Picker: &idlePicker{subConn: subConn}, + }) case connectivity.TransientFailure: b.cc.UpdateState(balancer.State{ - ConnectivityState: s.ConnectivityState, - Picker: &picker{err: s.ConnectionError}, + ConnectivityState: state.ConnectivityState, + Picker: &picker{err: state.ConnectionError}, }) } } @@ -125,8 +153,8 @@ func (b *pickfirstBalancer) Close() { } func (b *pickfirstBalancer) ExitIdle() { - if b.sc != nil && b.state == connectivity.Idle { - b.sc.Connect() + if b.subConn != nil && b.state == connectivity.Idle { + b.subConn.Connect() } } @@ -135,18 +163,18 @@ type picker struct { err error } -func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { +func (p *picker) Pick(balancer.PickInfo) (balancer.PickResult, error) { return p.result, p.err } // idlePicker is used when the SubConn is IDLE and kicks the SubConn into // CONNECTING when Pick is called. type idlePicker struct { - sc balancer.SubConn + subConn balancer.SubConn } -func (i *idlePicker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { - i.sc.Connect() +func (i *idlePicker) Pick(balancer.PickInfo) (balancer.PickResult, error) { + i.subConn.Connect() return balancer.PickResult{}, balancer.ErrNoSubConnAvailable } diff --git a/vendor/google.golang.org/grpc/resolver/resolver.go b/vendor/google.golang.org/grpc/resolver/resolver.go index e28b68026..ca2e35a35 100644 --- a/vendor/google.golang.org/grpc/resolver/resolver.go +++ b/vendor/google.golang.org/grpc/resolver/resolver.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/attributes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/serviceconfig" ) @@ -139,13 +140,18 @@ type Address struct { // Equal returns whether a and o are identical. Metadata is compared directly, // not with any recursive introspection. -func (a *Address) Equal(o Address) bool { +func (a Address) Equal(o Address) bool { return a.Addr == o.Addr && a.ServerName == o.ServerName && a.Attributes.Equal(o.Attributes) && a.BalancerAttributes.Equal(o.BalancerAttributes) && a.Type == o.Type && a.Metadata == o.Metadata } +// String returns JSON formatted string representation of the address. +func (a Address) String() string { + return pretty.ToJSON(a) +} + // BuildOptions includes additional information for the builder to create // the resolver. type BuildOptions struct { diff --git a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go index 2c47cd54f..05a9d4e0b 100644 --- a/vendor/google.golang.org/grpc/resolver_conn_wrapper.go +++ b/vendor/google.golang.org/grpc/resolver_conn_wrapper.go @@ -19,7 +19,6 @@ package grpc import ( - "fmt" "strings" "sync" @@ -27,6 +26,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcsync" + "google.golang.org/grpc/internal/pretty" "google.golang.org/grpc/resolver" "google.golang.org/grpc/serviceconfig" ) @@ -97,10 +97,7 @@ func (ccr *ccResolverWrapper) UpdateState(s resolver.State) error { if ccr.done.HasFired() { return nil } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending update to cc: %v", s) - if channelz.IsOn() { - ccr.addChannelzTraceEvent(s) - } + ccr.addChannelzTraceEvent(s) ccr.curState = s if err := ccr.cc.updateResolverState(ccr.curState, nil); err == balancer.ErrBadResolverState { return balancer.ErrBadResolverState @@ -125,10 +122,7 @@ func (ccr *ccResolverWrapper) NewAddress(addrs []resolver.Address) { if ccr.done.HasFired() { return } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: sending new addresses to cc: %v", addrs) - if channelz.IsOn() { - ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig}) - } + ccr.addChannelzTraceEvent(resolver.State{Addresses: addrs, ServiceConfig: ccr.curState.ServiceConfig}) ccr.curState.Addresses = addrs ccr.cc.updateResolverState(ccr.curState, nil) } @@ -141,7 +135,7 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { if ccr.done.HasFired() { return } - channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: got new service config: %v", sc) + channelz.Infof(logger, ccr.cc.channelzID, "ccResolverWrapper: got new service config: %s", sc) if ccr.cc.dopts.disableServiceConfig { channelz.Info(logger, ccr.cc.channelzID, "Service config lookups disabled; ignoring config") return @@ -151,9 +145,7 @@ func (ccr *ccResolverWrapper) NewServiceConfig(sc string) { channelz.Warningf(logger, ccr.cc.channelzID, "ccResolverWrapper: error parsing service config: %v", scpr.Err) return } - if channelz.IsOn() { - ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: scpr}) - } + ccr.addChannelzTraceEvent(resolver.State{Addresses: ccr.curState.Addresses, ServiceConfig: scpr}) ccr.curState.ServiceConfig = scpr ccr.cc.updateResolverState(ccr.curState, nil) } @@ -180,8 +172,5 @@ func (ccr *ccResolverWrapper) addChannelzTraceEvent(s resolver.State) { } else if len(ccr.curState.Addresses) == 0 && len(s.Addresses) > 0 { updates = append(updates, "resolver returned new addresses") } - channelz.AddTraceEvent(logger, ccr.cc.channelzID, 0, &channelz.TraceEventDesc{ - Desc: fmt.Sprintf("Resolver state updated: %+v (%v)", s, strings.Join(updates, "; ")), - Severity: channelz.CtInfo, - }) + channelz.Infof(logger, ccr.cc.channelzID, "Resolver state updated: %s (%v)", pretty.ToJSON(s), strings.Join(updates, "; ")) } diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index b24b6d539..96431a058 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -134,7 +134,7 @@ type Server struct { channelzRemoveOnce sync.Once serveWG sync.WaitGroup // counts active Serve goroutines for GracefulStop - channelzID int64 // channelz unique identification number + channelzID *channelz.Identifier czData *channelzData serverWorkerChannels []chan *serverWorkerData @@ -584,9 +584,8 @@ func NewServer(opt ...ServerOption) *Server { s.initServerWorkers() } - if channelz.IsOn() { - s.channelzID = channelz.RegisterServer(&channelzServer{s}, "") - } + s.channelzID = channelz.RegisterServer(&channelzServer{s}, "") + channelz.Info(logger, s.channelzID, "Server created") return s } @@ -712,7 +711,7 @@ var ErrServerStopped = errors.New("grpc: the server has been stopped") type listenSocket struct { net.Listener - channelzID int64 + channelzID *channelz.Identifier } func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { @@ -724,9 +723,8 @@ func (l *listenSocket) ChannelzMetric() *channelz.SocketInternalMetric { func (l *listenSocket) Close() error { err := l.Listener.Close() - if channelz.IsOn() { - channelz.RemoveEntry(l.channelzID) - } + channelz.RemoveEntry(l.channelzID) + channelz.Info(logger, l.channelzID, "ListenSocket deleted") return err } @@ -759,11 +757,6 @@ func (s *Server) Serve(lis net.Listener) error { ls := &listenSocket{Listener: lis} s.lis[ls] = true - if channelz.IsOn() { - ls.channelzID = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) - } - s.mu.Unlock() - defer func() { s.mu.Lock() if s.lis != nil && s.lis[ls] { @@ -773,8 +766,16 @@ func (s *Server) Serve(lis net.Listener) error { s.mu.Unlock() }() - var tempDelay time.Duration // how long to sleep on accept failure + var err error + ls.channelzID, err = channelz.RegisterListenSocket(ls, s.channelzID, lis.Addr().String()) + if err != nil { + s.mu.Unlock() + return err + } + s.mu.Unlock() + channelz.Info(logger, ls.channelzID, "ListenSocket created") + var tempDelay time.Duration // how long to sleep on accept failure for { rawConn, err := lis.Accept() if err != nil { @@ -1709,11 +1710,7 @@ func (s *Server) Stop() { s.done.Fire() }() - s.channelzRemoveOnce.Do(func() { - if channelz.IsOn() { - channelz.RemoveEntry(s.channelzID) - } - }) + s.channelzRemoveOnce.Do(func() { channelz.RemoveEntry(s.channelzID) }) s.mu.Lock() listeners := s.lis @@ -1751,11 +1748,7 @@ func (s *Server) GracefulStop() { s.quit.Fire() defer s.done.Fire() - s.channelzRemoveOnce.Do(func() { - if channelz.IsOn() { - channelz.RemoveEntry(s.channelzID) - } - }) + s.channelzRemoveOnce.Do(func() { channelz.RemoveEntry(s.channelzID) }) s.mu.Lock() if s.conns == nil { s.mu.Unlock() diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go index 6926a06dc..b01c548bb 100644 --- a/vendor/google.golang.org/grpc/service_config.go +++ b/vendor/google.golang.org/grpc/service_config.go @@ -381,6 +381,9 @@ func init() { // // If any of them is NOT *ServiceConfig, return false. func equalServiceConfig(a, b serviceconfig.Config) bool { + if a == nil && b == nil { + return true + } aa, ok := a.(*ServiceConfig) if !ok { return false diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 8cdd652e0..e0b30b46f 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -36,6 +36,7 @@ import ( "google.golang.org/grpc/internal/channelz" "google.golang.org/grpc/internal/grpcrand" "google.golang.org/grpc/internal/grpcutil" + imetadata "google.golang.org/grpc/internal/metadata" iresolver "google.golang.org/grpc/internal/resolver" "google.golang.org/grpc/internal/serviceconfig" "google.golang.org/grpc/internal/transport" @@ -166,6 +167,11 @@ func NewClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, meth } func newClientStream(ctx context.Context, desc *StreamDesc, cc *ClientConn, method string, opts ...CallOption) (_ ClientStream, err error) { + if md, _, ok := metadata.FromOutgoingContextRaw(ctx); ok { + if err := imetadata.Validate(md); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } if channelz.IsOn() { cc.incrCallsStarted() defer func() { @@ -456,7 +462,7 @@ type clientStream struct { retryThrottler *retryThrottler // The throttler active when the RPC began. - binlog *binarylog.MethodLogger // Binary logger, can be nil. + binlog binarylog.MethodLogger // Binary logger, can be nil. // serverHeaderBinlogged is a boolean for whether server header has been // logged. Server header will be logged when the first time one of those // happens: stream.Header(), stream.Recv(). @@ -1428,7 +1434,7 @@ type serverStream struct { statsHandler stats.Handler - binlog *binarylog.MethodLogger + binlog binarylog.MethodLogger // serverHeaderBinlogged indicates whether server header has been logged. It // will happen when one of the following two happens: stream.SendHeader(), // stream.Send(). @@ -1448,11 +1454,20 @@ func (ss *serverStream) SetHeader(md metadata.MD) error { if md.Len() == 0 { return nil } + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } return ss.s.SetHeader(md) } func (ss *serverStream) SendHeader(md metadata.MD) error { - err := ss.t.WriteHeader(ss.s, md) + err := imetadata.Validate(md) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + err = ss.t.WriteHeader(ss.s, md) if ss.binlog != nil && !ss.serverHeaderBinlogged { h, _ := ss.s.Header() ss.binlog.Log(&binarylog.ServerHeader{ @@ -1467,6 +1482,9 @@ func (ss *serverStream) SetTrailer(md metadata.MD) { if md.Len() == 0 { return } + if err := imetadata.Validate(md); err != nil { + logger.Errorf("stream: failed to validate md when setting trailer, err: %v", err) + } ss.s.SetTrailer(md) } diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index 5bd4f534c..6af76dfe7 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.45.0" +const Version = "1.46.0" diff --git a/vendor/k8s.io/api/batch/v1/types.go b/vendor/k8s.io/api/batch/v1/types.go index aec1cad8e..13cebde3f 100644 --- a/vendor/k8s.io/api/batch/v1/types.go +++ b/vendor/k8s.io/api/batch/v1/types.go @@ -289,10 +289,9 @@ type UncountedTerminatedPods struct { Failed []types.UID `json:"failed,omitempty" protobuf:"bytes,2,rep,name=failed,casttype=k8s.io/apimachinery/pkg/types.UID"` } -// +enum type JobConditionType string -// These are valid conditions of a job. +// These are built-in conditions of a job. const ( // JobSuspended means the job has been suspended. JobSuspended JobConditionType = "Suspended" diff --git a/vendor/k8s.io/api/certificates/v1/types.go b/vendor/k8s.io/api/certificates/v1/types.go index a3900e1dc..6403d4600 100644 --- a/vendor/k8s.io/api/certificates/v1/types.go +++ b/vendor/k8s.io/api/certificates/v1/types.go @@ -214,7 +214,6 @@ type CertificateSigningRequestStatus struct { } // RequestConditionType is the type of a CertificateSigningRequestCondition -// +enum type RequestConditionType string // Well-known condition types for certificate requests. diff --git a/vendor/k8s.io/api/core/v1/types.go b/vendor/k8s.io/api/core/v1/types.go index 80c5dd744..dcf83eccd 100644 --- a/vendor/k8s.io/api/core/v1/types.go +++ b/vendor/k8s.io/api/core/v1/types.go @@ -522,7 +522,6 @@ type PersistentVolumeClaimSpec struct { } // PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type -// +enum type PersistentVolumeClaimConditionType string const ( @@ -2635,10 +2634,9 @@ const ( ) // PodConditionType is a valid value for PodCondition.Type -// +enum type PodConditionType string -// These are valid conditions of pod. +// These are built-in conditions of pod. An application may use a custom condition not listed here. const ( // ContainersReady indicates whether all containers in the pod are ready. ContainersReady PodConditionType = "ContainersReady" @@ -5055,11 +5053,10 @@ const ( NodeTerminated NodePhase = "Terminated" ) -// +enum type NodeConditionType string -// These are valid conditions of node. Currently, we don't have enough information to decide -// node condition. In the future, we will add more. The proposed set of conditions are: +// These are valid but not exhaustive conditions of node. A cloud provider may set a condition not listed here. +// The built-in set of conditions are: // NodeReachable, NodeLive, NodeReady, NodeSchedulable, NodeRunnable. const ( // NodeReady means kubelet is healthy and ready to accept pods. @@ -5094,10 +5091,9 @@ type NodeCondition struct { Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"` } -// +enum type NodeAddressType string -// These are valid address type of node. +// These are built-in addresses type of node. A cloud provider may set a type not listed here. const ( // NodeHostName identifies a name of the node. Although every node can be assumed // to have a NodeAddress of this type, its exact syntax and semantics are not @@ -5267,10 +5263,9 @@ const ( NamespaceTerminatingCause metav1.CauseType = "NamespaceTerminating" ) -// +enum type NamespaceConditionType string -// These are valid conditions of a namespace. +// These are built-in conditions of a namespace. const ( // NamespaceDeletionDiscoveryFailure contains information about namespace deleter errors during resource discovery. NamespaceDeletionDiscoveryFailure NamespaceConditionType = "NamespaceDeletionDiscoveryFailure" @@ -5758,8 +5753,8 @@ type EventList struct { // List holds a list of objects, which may not be known by the server. type List metav1.List -// LimitType is a type of object that is limited -// +enum +// LimitType is a type of object that is limited. It can be Pod, Container, PersistentVolumeClaim or +// a fully qualified resource name. type LimitType string const ( diff --git a/vendor/k8s.io/client-go/kubernetes/clientset.go b/vendor/k8s.io/client-go/kubernetes/clientset.go index 3e512a7c2..e46c0537f 100644 --- a/vendor/k8s.io/client-go/kubernetes/clientset.go +++ b/vendor/k8s.io/client-go/kubernetes/clientset.go @@ -413,6 +413,10 @@ func (c *Clientset) Discovery() discovery.DiscoveryInterface { func NewForConfig(c *rest.Config) (*Clientset, error) { configShallowCopy := *c + if configShallowCopy.UserAgent == "" { + configShallowCopy.UserAgent = rest.DefaultKubernetesUserAgent() + } + // share the transport between all clients httpClient, err := rest.HTTPClientFor(&configShallowCopy) if err != nil { diff --git a/vendor/k8s.io/client-go/pkg/version/.gitattributes b/vendor/k8s.io/client-go/pkg/version/.gitattributes deleted file mode 100644 index 7e349eff6..000000000 --- a/vendor/k8s.io/client-go/pkg/version/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -base.go export-subst diff --git a/vendor/k8s.io/client-go/tools/cache/thread_safe_store.go b/vendor/k8s.io/client-go/tools/cache/thread_safe_store.go index ea34e9035..71d909d49 100644 --- a/vendor/k8s.io/client-go/tools/cache/thread_safe_store.go +++ b/vendor/k8s.io/client-go/tools/cache/thread_safe_store.go @@ -284,18 +284,15 @@ func (c *threadSafeMap) updateIndices(oldObj interface{}, newObj interface{}, ke c.indices[name] = index } + if len(indexValues) == 1 && len(oldIndexValues) == 1 && indexValues[0] == oldIndexValues[0] { + // We optimize for the most common case where indexFunc returns a single value which has not been changed + continue + } + for _, value := range oldIndexValues { - // We optimize for the most common case where index returns a single value. - if len(indexValues) == 1 && value == indexValues[0] { - continue - } c.deleteKeyFromIndex(key, value, index) } for _, value := range indexValues { - // We optimize for the most common case where index returns a single value. - if len(oldIndexValues) == 1 && value == oldIndexValues[0] { - continue - } c.addKeyToIndex(key, value, index) } } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go b/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go index 5d72e89f7..6472b8919 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/framework.go @@ -248,6 +248,9 @@ func (f *Framework) BeforeEach() { ginkgo.By("Waiting for a default service account to be provisioned in namespace") err = WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name) ExpectNoError(err) + ginkgo.By("Waiting for kube-root-ca.crt to be provisioned in namespace") + err = WaitForKubeRootCAInNamespace(f.ClientSet, namespace.Name) + ExpectNoError(err) } else { Logf("Skipping waiting for service account") } diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/util.go b/vendor/k8s.io/kubernetes/test/e2e/framework/util.go index 3f3602e9f..137dcaa8c 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/util.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/util.go @@ -282,6 +282,32 @@ func WaitForNamespacesDeleted(c clientset.Interface, namespaces []string, timeou }) } +func waitForConfigMapInNamespace(c clientset.Interface, ns, name string, timeout time.Duration) error { + fieldSelector := fields.OneTermEqualSelector("metadata.name", name).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) { + options.FieldSelector = fieldSelector + return c.CoreV1().ConfigMaps(ns).List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) { + options.FieldSelector = fieldSelector + return c.CoreV1().ConfigMaps(ns).Watch(context.TODO(), options) + }, + } + ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) + defer cancel() + _, err := watchtools.UntilWithSync(ctx, lw, &v1.ConfigMap{}, nil, func(event watch.Event) (bool, error) { + switch event.Type { + case watch.Deleted: + return false, apierrors.NewNotFound(schema.GroupResource{Resource: "configmaps"}, name) + case watch.Added, watch.Modified: + return true, nil + } + return false, nil + }) + return err +} + func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountName string, timeout time.Duration) error { fieldSelector := fields.OneTermEqualSelector("metadata.name", serviceAccountName).String() lw := &cache.ListWatch{ @@ -296,24 +322,18 @@ func waitForServiceAccountInNamespace(c clientset.Interface, ns, serviceAccountN } ctx, cancel := watchtools.ContextWithOptionalTimeout(context.Background(), timeout) defer cancel() - _, err := watchtools.UntilWithSync(ctx, lw, &v1.ServiceAccount{}, nil, serviceAccountHasSecrets) + _, err := watchtools.UntilWithSync(ctx, lw, &v1.ServiceAccount{}, nil, func(event watch.Event) (bool, error) { + switch event.Type { + case watch.Deleted: + return false, apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, serviceAccountName) + case watch.Added, watch.Modified: + return true, nil + } + return false, nil + }) return err } -// serviceAccountHasSecrets returns true if the service account has at least one secret, -// false if it does not, or an error. -func serviceAccountHasSecrets(event watch.Event) (bool, error) { - switch event.Type { - case watch.Deleted: - return false, apierrors.NewNotFound(schema.GroupResource{Resource: "serviceaccounts"}, "") - } - switch t := event.Object.(type) { - case *v1.ServiceAccount: - return len(t.Secrets) > 0, nil - } - return false, nil -} - // WaitForDefaultServiceAccountInNamespace waits for the default service account to be provisioned // the default service account is what is associated with pods when they do not specify a service account // as a result, pods are not able to be provisioned in a namespace until the service account is provisioned @@ -321,6 +341,13 @@ func WaitForDefaultServiceAccountInNamespace(c clientset.Interface, namespace st return waitForServiceAccountInNamespace(c, namespace, "default", ServiceAccountProvisionTimeout) } +// WaitForKubeRootCAInNamespace waits for the configmap kube-root-ca.crt containing the service account +// CA trust bundle to be provisioned in the specified namespace so that pods do not have to retry mounting +// the config map (which creates noise that hides other issues in the Kubelet). +func WaitForKubeRootCAInNamespace(c clientset.Interface, namespace string) error { + return waitForConfigMapInNamespace(c, namespace, "kube-root-ca.crt", ServiceAccountProvisionTimeout) +} + // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. // Please see NewFramework instead of using this directly. func CreateTestingNS(baseName string, c clientset.Interface, labels map[string]string) (*v1.Namespace, error) { diff --git a/vendor/modules.txt b/vendor/modules.txt index d2d7e8949..ab4443cac 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -8,7 +8,7 @@ github.com/armon/go-metrics # github.com/armon/go-radix v1.0.0 ## explicit github.com/armon/go-radix -# github.com/aws/aws-sdk-go v1.43.32 +# github.com/aws/aws-sdk-go v1.43.41 ## explicit; go 1.11 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr @@ -53,7 +53,7 @@ github.com/aws/aws-sdk-go/service/sso github.com/aws/aws-sdk-go/service/sso/ssoiface github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface -# github.com/aws/aws-sdk-go-v2 v1.16.2 +# github.com/aws/aws-sdk-go-v2 v1.16.3 ## explicit; go 1.15 github.com/aws/aws-sdk-go-v2/aws github.com/aws/aws-sdk-go-v2/aws/defaults @@ -70,16 +70,16 @@ github.com/aws/aws-sdk-go-v2/internal/sdk github.com/aws/aws-sdk-go-v2/internal/strings github.com/aws/aws-sdk-go-v2/internal/sync/singleflight github.com/aws/aws-sdk-go-v2/internal/timeconv -# github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 +# github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 ## explicit; go 1.15 github.com/aws/aws-sdk-go-v2/internal/configsources -# github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 +# github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 ## explicit; go 1.15 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 -# github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 +# github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 ## explicit; go 1.15 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url -# github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 +# github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 ## explicit; go 1.15 github.com/aws/aws-sdk-go-v2/service/sts github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints @@ -135,7 +135,7 @@ github.com/ceph/go-ceph/rbd/admin # github.com/cespare/xxhash/v2 v2.1.2 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/container-storage-interface/spec v1.5.0 +# github.com/container-storage-interface/spec v1.6.0 ## explicit; go 1.16 github.com/container-storage-interface/spec/lib/go/csi # github.com/csi-addons/replication-lib-utils v0.2.0 @@ -639,7 +639,7 @@ google.golang.org/appengine/urlfetch google.golang.org/genproto/googleapis/api/httpbody google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/protobuf/field_mask -# google.golang.org/grpc v1.45.0 +# google.golang.org/grpc v1.46.0 ## explicit; go 1.14 google.golang.org/grpc google.golang.org/grpc/attributes @@ -649,6 +649,7 @@ google.golang.org/grpc/balancer/base google.golang.org/grpc/balancer/grpclb/state google.golang.org/grpc/balancer/roundrobin google.golang.org/grpc/binarylog/grpc_binarylog_v1 +google.golang.org/grpc/channelz google.golang.org/grpc/codes google.golang.org/grpc/connectivity google.golang.org/grpc/credentials @@ -661,6 +662,7 @@ google.golang.org/grpc/health google.golang.org/grpc/health/grpc_health_v1 google.golang.org/grpc/internal google.golang.org/grpc/internal/backoff +google.golang.org/grpc/internal/balancer/gracefulswitch google.golang.org/grpc/internal/balancerload google.golang.org/grpc/internal/binarylog google.golang.org/grpc/internal/buffer @@ -672,6 +674,7 @@ google.golang.org/grpc/internal/grpcrand google.golang.org/grpc/internal/grpcsync google.golang.org/grpc/internal/grpcutil google.golang.org/grpc/internal/metadata +google.golang.org/grpc/internal/pretty google.golang.org/grpc/internal/resolver google.golang.org/grpc/internal/resolver/dns google.golang.org/grpc/internal/resolver/passthrough @@ -746,7 +749,7 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ## explicit gopkg.in/yaml.v3 -# k8s.io/api v0.23.5 => k8s.io/api v0.23.5 +# k8s.io/api v0.23.6 => k8s.io/api v0.23.6 ## explicit; go 1.16 k8s.io/api/admission/v1 k8s.io/api/admission/v1beta1 @@ -795,7 +798,7 @@ k8s.io/api/scheduling/v1beta1 k8s.io/api/storage/v1 k8s.io/api/storage/v1alpha1 k8s.io/api/storage/v1beta1 -# k8s.io/apimachinery v0.23.5 => k8s.io/apimachinery v0.23.5 +# k8s.io/apimachinery v0.23.6 => k8s.io/apimachinery v0.23.6 ## explicit; go 1.16 k8s.io/apimachinery/pkg/api/equality k8s.io/apimachinery/pkg/api/errors @@ -851,7 +854,7 @@ k8s.io/apimachinery/pkg/watch k8s.io/apimachinery/third_party/forked/golang/json k8s.io/apimachinery/third_party/forked/golang/netutil k8s.io/apimachinery/third_party/forked/golang/reflect -# k8s.io/apiserver v0.23.5 => k8s.io/apiserver v0.23.5 +# k8s.io/apiserver v0.23.6 => k8s.io/apiserver v0.23.6 ## explicit; go 1.16 k8s.io/apiserver/pkg/admission k8s.io/apiserver/pkg/admission/configuration @@ -892,7 +895,7 @@ k8s.io/apiserver/pkg/util/feature k8s.io/apiserver/pkg/util/webhook k8s.io/apiserver/pkg/util/x509metrics k8s.io/apiserver/pkg/warning -# k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.23.5 +# k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.23.6 ## explicit; go 1.16 k8s.io/client-go/applyconfigurations/admissionregistration/v1 k8s.io/client-go/applyconfigurations/admissionregistration/v1beta1 @@ -1135,12 +1138,12 @@ k8s.io/client-go/util/homedir k8s.io/client-go/util/keyutil k8s.io/client-go/util/retry k8s.io/client-go/util/workqueue -# k8s.io/cloud-provider v0.23.5 => k8s.io/cloud-provider v0.23.5 +# k8s.io/cloud-provider v0.23.6 => k8s.io/cloud-provider v0.23.6 ## explicit; go 1.16 k8s.io/cloud-provider k8s.io/cloud-provider/volume k8s.io/cloud-provider/volume/helpers -# k8s.io/component-base v0.23.5 => k8s.io/component-base v0.23.5 +# k8s.io/component-base v0.23.6 => k8s.io/component-base v0.23.6 ## explicit; go 1.16 k8s.io/component-base/cli/flag k8s.io/component-base/config @@ -1151,7 +1154,7 @@ k8s.io/component-base/metrics/legacyregistry k8s.io/component-base/metrics/testutil k8s.io/component-base/traces k8s.io/component-base/version -# k8s.io/component-helpers v0.23.5 => k8s.io/component-helpers v0.23.5 +# k8s.io/component-helpers v0.23.6 => k8s.io/component-helpers v0.23.6 ## explicit; go 1.16 k8s.io/component-helpers/node/util/sysctl k8s.io/component-helpers/scheduling/corev1 @@ -1167,14 +1170,14 @@ k8s.io/klog/v2/internal/severity ## explicit; go 1.16 k8s.io/kube-openapi/pkg/schemaconv k8s.io/kube-openapi/pkg/util/proto -# k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.23.5 +# k8s.io/kubectl v0.0.0 => k8s.io/kubectl v0.23.6 ## explicit; go 1.16 k8s.io/kubectl/pkg/scale k8s.io/kubectl/pkg/util/podutils -# k8s.io/kubelet v0.0.0 => k8s.io/kubelet v0.23.5 +# k8s.io/kubelet v0.0.0 => k8s.io/kubelet v0.23.6 ## explicit; go 1.16 k8s.io/kubelet/pkg/apis/stats/v1alpha1 -# k8s.io/kubernetes v1.23.5 +# k8s.io/kubernetes v1.23.6 ## explicit; go 1.16 k8s.io/kubernetes/pkg/api/legacyscheme k8s.io/kubernetes/pkg/api/service @@ -1237,7 +1240,7 @@ k8s.io/kubernetes/test/e2e/storage/podlogs k8s.io/kubernetes/test/e2e/storage/utils k8s.io/kubernetes/test/utils k8s.io/kubernetes/test/utils/image -# k8s.io/mount-utils v0.23.5 => k8s.io/mount-utils v0.23.5 +# k8s.io/mount-utils v0.23.6 => k8s.io/mount-utils v0.23.6 ## explicit; go 1.16 k8s.io/mount-utils # k8s.io/utils v0.0.0-20211116205334-6203023598ed @@ -1316,29 +1319,29 @@ sigs.k8s.io/yaml # github.com/golang/protobuf => github.com/golang/protobuf v1.4.3 # github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 # gomodules.xyz/jsonpatch/v2 => github.com/gomodules/jsonpatch/v2 v2.2.0 -# k8s.io/api => k8s.io/api v0.23.5 -# k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.5 -# k8s.io/apimachinery => k8s.io/apimachinery v0.23.5 -# k8s.io/apiserver => k8s.io/apiserver v0.23.5 -# k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.5 -# k8s.io/client-go => k8s.io/client-go v0.23.5 -# k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.5 -# k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.5 -# k8s.io/code-generator => k8s.io/code-generator v0.23.5 -# k8s.io/component-base => k8s.io/component-base v0.23.5 -# k8s.io/component-helpers => k8s.io/component-helpers v0.23.5 -# k8s.io/controller-manager => k8s.io/controller-manager v0.23.5 -# k8s.io/cri-api => k8s.io/cri-api v0.23.5 -# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.5 -# k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.5 -# k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.5 -# k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.5 -# k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.5 -# k8s.io/kubectl => k8s.io/kubectl v0.23.5 -# k8s.io/kubelet => k8s.io/kubelet v0.23.5 -# k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.5 -# k8s.io/metrics => k8s.io/metrics v0.23.5 -# k8s.io/mount-utils => k8s.io/mount-utils v0.23.5 -# k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.5 -# k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.5 +# k8s.io/api => k8s.io/api v0.23.6 +# k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.6 +# k8s.io/apimachinery => k8s.io/apimachinery v0.23.6 +# k8s.io/apiserver => k8s.io/apiserver v0.23.6 +# k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.6 +# k8s.io/client-go => k8s.io/client-go v0.23.6 +# k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.6 +# k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.6 +# k8s.io/code-generator => k8s.io/code-generator v0.23.6 +# k8s.io/component-base => k8s.io/component-base v0.23.6 +# k8s.io/component-helpers => k8s.io/component-helpers v0.23.6 +# k8s.io/controller-manager => k8s.io/controller-manager v0.23.6 +# k8s.io/cri-api => k8s.io/cri-api v0.23.6 +# k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.6 +# k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.6 +# k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.6 +# k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.6 +# k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.6 +# k8s.io/kubectl => k8s.io/kubectl v0.23.6 +# k8s.io/kubelet => k8s.io/kubelet v0.23.6 +# k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.6 +# k8s.io/metrics => k8s.io/metrics v0.23.6 +# k8s.io/mount-utils => k8s.io/mount-utils v0.23.6 +# k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.6 +# k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.6 # layeh.com/radius => github.com/layeh/radius v0.0.0-20190322222518-890bc1058917