From 038462ff43b3898bde3d19f783d1c73c956d88e1 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 29 Aug 2022 11:03:18 +0530 Subject: [PATCH 01/11] cephfs: return success if metadata operation not supported If the ceph cluster is of older version and doesnot support metadata operation, Instead of failing the request return the success if metadata operation is not supported. fixes #3347 Signed-off-by: Madhu Rajanna --- internal/cephfs/core/metadata.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/cephfs/core/metadata.go b/internal/cephfs/core/metadata.go index d124d2eda..e0cedbd72 100644 --- a/internal/cephfs/core/metadata.go +++ b/internal/cephfs/core/metadata.go @@ -99,6 +99,10 @@ func (s *subVolumeClient) SetAllMetadata(parameters map[string]string) error { for k, v := range parameters { err := s.setMetadata(k, v) + // If setMetadata is not supported return nil + if errors.Is(err, ErrSubVolMetadataNotSupported) { + return nil + } if err != nil { return fmt.Errorf("failed to set metadata key %q, value %q on subvolume %v: %w", k, v, s, err) } @@ -106,6 +110,10 @@ func (s *subVolumeClient) SetAllMetadata(parameters map[string]string) error { if s.clusterName != "" { err := s.setMetadata(clusterNameKey, s.clusterName) + // If setMetadata is not supported return nil + if errors.Is(err, ErrSubVolMetadataNotSupported) { + return nil + } if err != nil { return fmt.Errorf("failed to set metadata key %q, value %q on subvolume %v: %w", clusterNameKey, s.clusterName, s, err) @@ -123,6 +131,10 @@ func (s *subVolumeClient) UnsetAllMetadata(keys []string) error { for _, key := range keys { err := s.removeMetadata(key) + // If setMetadata is not supported return nil + if errors.Is(err, ErrSubVolMetadataNotSupported) { + return nil + } // TODO: replace string comparison with errno. if err != nil && !strings.Contains(err.Error(), "No such file or directory") { return fmt.Errorf("failed to unset metadata key %q on subvolume %v: %w", key, s, err) @@ -130,6 +142,10 @@ func (s *subVolumeClient) UnsetAllMetadata(keys []string) error { } err := s.removeMetadata(clusterNameKey) + // If setMetadata is not supported return nil + if errors.Is(err, ErrSubVolMetadataNotSupported) { + return nil + } // TODO: replace string comparison with errno. if err != nil && !strings.Contains(err.Error(), "No such file or directory") { return fmt.Errorf("failed to unset metadata key %q on subvolume %v: %w", clusterNameKey, s, err) From 586a9cc8eeed2ca777f1a51b979852d39dd840b5 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Thu, 1 Sep 2022 09:44:25 +0530 Subject: [PATCH 02/11] rbd: change default FsGroupPolicy to "File" for RBD CSI driver This commit change the default fsgroup policy for csi driver object to "File" type which is the better/correct setting for the CSI volumes. We have been using default value which is "ReadWriteOnceWithFSType". with this change backward compatibility should be preserved. Signed-off-by: Humble Chirammal --- charts/ceph-csi-rbd/templates/csidriver-crd.yaml | 1 + deploy/rbd/kubernetes/csidriver.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/charts/ceph-csi-rbd/templates/csidriver-crd.yaml b/charts/ceph-csi-rbd/templates/csidriver-crd.yaml index 92c3d916a..f4340a573 100644 --- a/charts/ceph-csi-rbd/templates/csidriver-crd.yaml +++ b/charts/ceph-csi-rbd/templates/csidriver-crd.yaml @@ -9,3 +9,4 @@ metadata: spec: attachRequired: true podInfoOnMount: false + fsGroupPolicy: File diff --git a/deploy/rbd/kubernetes/csidriver.yaml b/deploy/rbd/kubernetes/csidriver.yaml index 1ece4e96e..77dfac197 100644 --- a/deploy/rbd/kubernetes/csidriver.yaml +++ b/deploy/rbd/kubernetes/csidriver.yaml @@ -15,3 +15,4 @@ metadata: spec: attachRequired: true podInfoOnMount: false + fsGroupPolicy: File From 9a54f05e663cd92583bd6a6089122f562823f8d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 20:15:30 +0000 Subject: [PATCH 03/11] rebase: bump google.golang.org/grpc from 1.48.0 to 1.49.0 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.48.0 to 1.49.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.48.0...v1.49.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 +- .../grpc/balancer/balancer.go | 53 -------------- .../grpc/balancer/conn_state_evaluator.go | 70 +++++++++++++++++++ vendor/google.golang.org/grpc/clientconn.go | 3 +- vendor/google.golang.org/grpc/dialoptions.go | 25 ++++--- .../grpc/internal/envconfig/xds.go | 3 +- .../grpc/internal/grpcutil/method.go | 5 ++ .../grpc/internal/transport/http2_client.go | 30 ++++---- .../grpc/internal/transport/http2_server.go | 21 +++--- .../grpc/internal/transport/http_util.go | 6 -- vendor/google.golang.org/grpc/server.go | 52 ++++++++------ .../google.golang.org/grpc/service_config.go | 7 +- vendor/google.golang.org/grpc/stream.go | 32 +++++---- vendor/google.golang.org/grpc/version.go | 2 +- vendor/google.golang.org/grpc/vet.sh | 1 - vendor/modules.txt | 4 +- 17 files changed, 175 insertions(+), 145 deletions(-) create mode 100644 vendor/google.golang.org/grpc/balancer/conn_state_evaluator.go diff --git a/go.mod b/go.mod index 321e1cee7..f34bfb2dc 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd golang.org/x/net v0.0.0-20220722155237-a158d28d115b golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f - google.golang.org/grpc v1.48.0 + google.golang.org/grpc v1.49.0 google.golang.org/protobuf v1.28.0 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 diff --git a/go.sum b/go.sum index 382ae58e6..eb7990c36 100644 --- a/go.sum +++ b/go.sum @@ -1761,8 +1761,8 @@ google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzI google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= +google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= diff --git a/vendor/google.golang.org/grpc/balancer/balancer.go b/vendor/google.golang.org/grpc/balancer/balancer.go index f7a7697ca..257139080 100644 --- a/vendor/google.golang.org/grpc/balancer/balancer.go +++ b/vendor/google.golang.org/grpc/balancer/balancer.go @@ -371,56 +371,3 @@ type ClientConnState struct { // ErrBadResolverState may be returned by UpdateClientConnState to indicate a // problem with the provided name resolver data. var ErrBadResolverState = errors.New("bad resolver state") - -// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns -// and returns one aggregated connectivity state. -// -// It's not thread safe. -type ConnectivityStateEvaluator struct { - numReady uint64 // Number of addrConns in ready state. - numConnecting uint64 // Number of addrConns in connecting state. - numTransientFailure uint64 // Number of addrConns in transient failure state. - numIdle uint64 // Number of addrConns in idle state. -} - -// RecordTransition records state change happening in subConn and based on that -// it evaluates what aggregated state should be. -// -// - If at least one SubConn in Ready, the aggregated state is Ready; -// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; -// - Else if at least one SubConn is TransientFailure, the aggregated state is Transient Failure; -// - Else if at least one SubConn is Idle, the aggregated state is Idle; -// - Else there are no subconns and the aggregated state is Transient Failure -// -// Shutdown is not considered. -func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { - // Update counters. - for idx, state := range []connectivity.State{oldState, newState} { - updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. - switch state { - case connectivity.Ready: - cse.numReady += updateVal - case connectivity.Connecting: - cse.numConnecting += updateVal - case connectivity.TransientFailure: - cse.numTransientFailure += updateVal - case connectivity.Idle: - cse.numIdle += updateVal - } - } - - // Evaluate. - if cse.numReady > 0 { - return connectivity.Ready - } - if cse.numConnecting > 0 { - return connectivity.Connecting - } - if cse.numTransientFailure > 0 { - return connectivity.TransientFailure - } - if cse.numIdle > 0 { - return connectivity.Idle - } - return connectivity.TransientFailure -} diff --git a/vendor/google.golang.org/grpc/balancer/conn_state_evaluator.go b/vendor/google.golang.org/grpc/balancer/conn_state_evaluator.go new file mode 100644 index 000000000..a87b6809a --- /dev/null +++ b/vendor/google.golang.org/grpc/balancer/conn_state_evaluator.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2022 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancer + +import "google.golang.org/grpc/connectivity" + +// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type ConnectivityStateEvaluator struct { + numReady uint64 // Number of addrConns in ready state. + numConnecting uint64 // Number of addrConns in connecting state. + numTransientFailure uint64 // Number of addrConns in transient failure state. + numIdle uint64 // Number of addrConns in idle state. +} + +// RecordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If at least one SubConn in Ready, the aggregated state is Ready; +// - Else if at least one SubConn in Connecting, the aggregated state is Connecting; +// - Else if at least one SubConn is Idle, the aggregated state is Idle; +// - Else if at least one SubConn is TransientFailure (or there are no SubConns), the aggregated state is Transient Failure. +// +// Shutdown is not considered. +func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + switch state { + case connectivity.Ready: + cse.numReady += updateVal + case connectivity.Connecting: + cse.numConnecting += updateVal + case connectivity.TransientFailure: + cse.numTransientFailure += updateVal + case connectivity.Idle: + cse.numIdle += updateVal + } + } + + // Evaluate. + if cse.numReady > 0 { + return connectivity.Ready + } + if cse.numConnecting > 0 { + return connectivity.Connecting + } + if cse.numIdle > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} diff --git a/vendor/google.golang.org/grpc/clientconn.go b/vendor/google.golang.org/grpc/clientconn.go index 0d21f2210..779b03bca 100644 --- a/vendor/google.golang.org/grpc/clientconn.go +++ b/vendor/google.golang.org/grpc/clientconn.go @@ -712,8 +712,8 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub ac.ctx, ac.cancel = context.WithCancel(cc.ctx) // Track ac in cc. This needs to be done before any getTransport(...) is called. cc.mu.Lock() + defer cc.mu.Unlock() if cc.conns == nil { - cc.mu.Unlock() return nil, ErrClientConnClosing } @@ -732,7 +732,6 @@ func (cc *ClientConn) newAddrConn(addrs []resolver.Address, opts balancer.NewSub }) cc.conns[ac] = struct{}{} - cc.mu.Unlock() return ac, nil } diff --git a/vendor/google.golang.org/grpc/dialoptions.go b/vendor/google.golang.org/grpc/dialoptions.go index 75d01ba77..60403bc16 100644 --- a/vendor/google.golang.org/grpc/dialoptions.go +++ b/vendor/google.golang.org/grpc/dialoptions.go @@ -84,7 +84,7 @@ var extraDialOptions []DialOption // EmptyDialOption does not alter the dial configuration. It can be embedded in // another structure to build custom dial options. // -// Experimental +// # Experimental // // Notice: This type is EXPERIMENTAL and may be changed or removed in a // later release. @@ -275,7 +275,7 @@ func WithBlock() DialOption { // the context.DeadlineExceeded error. // Implies WithBlock() // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -304,7 +304,7 @@ func WithInsecure() DialOption { // WithNoProxy returns a DialOption which disables the use of proxies for this // ClientConn. This is ignored if WithDialer or WithContextDialer are used. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -335,7 +335,7 @@ func WithPerRPCCredentials(creds credentials.PerRPCCredentials) DialOption { // the ClientConn.WithCreds. This should not be used together with // WithTransportCredentials. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -391,6 +391,12 @@ func WithDialer(f func(string, time.Duration) (net.Conn, error)) DialOption { // all the RPCs and underlying network connections in this ClientConn. func WithStatsHandler(h stats.Handler) DialOption { return newFuncDialOption(func(o *dialOptions) { + if h == nil { + logger.Error("ignoring nil parameter in grpc.WithStatsHandler ClientOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } o.copts.StatsHandlers = append(o.copts.StatsHandlers, h) }) } @@ -403,7 +409,7 @@ func WithStatsHandler(h stats.Handler) DialOption { // FailOnNonTempDialError only affects the initial dial, and does not do // anything useful unless you are also using WithBlock(). // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -483,7 +489,7 @@ func WithAuthority(a string) DialOption { // current ClientConn's parent. This function is used in nested channel creation // (e.g. grpclb dial). // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -528,9 +534,6 @@ func WithDefaultServiceConfig(s string) DialOption { // service config enables them. This does not impact transparent retries, which // will happen automatically if no data is written to the wire or if the RPC is // unprocessed by the remote server. -// -// Retry support is currently enabled by default, but may be disabled by -// setting the environment variable "GRPC_GO_RETRY" to "off". func WithDisableRetry() DialOption { return newFuncDialOption(func(o *dialOptions) { o.disableRetry = true @@ -548,7 +551,7 @@ func WithMaxHeaderListSize(s uint32) DialOption { // WithDisableHealthCheck disables the LB channel health checking for all // SubConns of this ClientConn. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -595,7 +598,7 @@ func withMinConnectDeadline(f func() time.Duration) DialOption { // resolver.Register. They will be matched against the scheme used for the // current Dial only, and will take precedence over the global registry. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. diff --git a/vendor/google.golang.org/grpc/internal/envconfig/xds.go b/vendor/google.golang.org/grpc/internal/envconfig/xds.go index 55aaeea8b..a83b26bb8 100644 --- a/vendor/google.golang.org/grpc/internal/envconfig/xds.go +++ b/vendor/google.golang.org/grpc/internal/envconfig/xds.go @@ -41,7 +41,6 @@ const ( clientSideSecuritySupportEnv = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT" aggregateAndDNSSupportEnv = "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER" rbacSupportEnv = "GRPC_XDS_EXPERIMENTAL_RBAC" - outlierDetectionSupportEnv = "GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION" federationEnv = "GRPC_EXPERIMENTAL_XDS_FEDERATION" rlsInXDSEnv = "GRPC_EXPERIMENTAL_XDS_RLS_LB" @@ -86,7 +85,7 @@ var ( // XDSOutlierDetection indicates whether outlier detection support is // enabled, which can be enabled by setting the environment variable // "GRPC_EXPERIMENTAL_ENABLE_OUTLIER_DETECTION" to "true". - XDSOutlierDetection = strings.EqualFold(os.Getenv(outlierDetectionSupportEnv), "true") + XDSOutlierDetection = false // XDSFederation indicates whether federation support is enabled. XDSFederation = strings.EqualFold(os.Getenv(federationEnv), "true") diff --git a/vendor/google.golang.org/grpc/internal/grpcutil/method.go b/vendor/google.golang.org/grpc/internal/grpcutil/method.go index 4e7475060..e9c4af648 100644 --- a/vendor/google.golang.org/grpc/internal/grpcutil/method.go +++ b/vendor/google.golang.org/grpc/internal/grpcutil/method.go @@ -39,6 +39,11 @@ func ParseMethod(methodName string) (service, method string, _ error) { return methodName[:pos], methodName[pos+1:], nil } +// baseContentType is the base content-type for gRPC. This is a valid +// content-type on it's own, but can also include a content-subtype such as +// "proto" as a suffix after "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests +// for more details. const baseContentType = "application/grpc" // ContentSubtype returns the content-subtype for the given content-type. The diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_client.go b/vendor/google.golang.org/grpc/internal/transport/http2_client.go index be371c6e0..28c77af70 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_client.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_client.go @@ -78,6 +78,7 @@ type http2Client struct { framer *framer // controlBuf delivers all the control related tasks (e.g., window // updates, reset streams, and various settings) to the controller. + // Do not access controlBuf with mu held. controlBuf *controlBuffer fc *trInFlow // The scheme used: https if TLS is on, http otherwise. @@ -109,6 +110,7 @@ type http2Client struct { waitingStreams uint32 nextID uint32 + // Do not access controlBuf with mu held. mu sync.Mutex // guard the following variables state transportState activeStreams map[uint32]*Stream @@ -685,7 +687,6 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, cleanup(err) return err } - t.activeStreams[id] = s if channelz.IsOn() { atomic.AddInt64(&t.czData.streamsStarted, 1) atomic.StoreInt64(&t.czData.lastStreamCreatedTime, time.Now().UnixNano()) @@ -719,6 +720,13 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, t.nextID += 2 s.id = h.streamID s.fc = &inFlow{limit: uint32(t.initialWindowSize)} + t.mu.Lock() + if t.activeStreams == nil { // Can be niled from Close(). + t.mu.Unlock() + return false // Don't create a stream if the transport is already closed. + } + t.activeStreams[s.id] = s + t.mu.Unlock() if t.streamQuota > 0 && t.waitingStreams > 0 { select { case t.streamsQuotaAvailable <- struct{}{}: @@ -744,13 +752,7 @@ func (t *http2Client) NewStream(ctx context.Context, callHdr *CallHdr) (*Stream, } for { success, err := t.controlBuf.executeAndPut(func(it interface{}) bool { - if !checkForStreamQuota(it) { - return false - } - if !checkForHeaderListSize(it) { - return false - } - return true + return checkForHeaderListSize(it) && checkForStreamQuota(it) }, hdr) if err != nil { // Connection closed. @@ -1003,13 +1005,13 @@ func (t *http2Client) updateWindow(s *Stream, n uint32) { // for the transport and the stream based on the current bdp // estimation. func (t *http2Client) updateFlowControl(n uint32) { - t.mu.Lock() - for _, s := range t.activeStreams { - s.fc.newLimit(n) - } - t.mu.Unlock() updateIWS := func(interface{}) bool { t.initialWindowSize = int32(n) + t.mu.Lock() + for _, s := range t.activeStreams { + s.fc.newLimit(n) + } + t.mu.Unlock() return true } t.controlBuf.executeAndPut(updateIWS, &outgoingWindowUpdate{streamID: 0, increment: t.fc.newLimit(n)}) @@ -1215,7 +1217,7 @@ func (t *http2Client) handleGoAway(f *http2.GoAwayFrame) { default: t.setGoAwayReason(f) close(t.goAway) - t.controlBuf.put(&incomingGoAway{}) + defer t.controlBuf.put(&incomingGoAway{}) // Defer as t.mu is currently held. // Notify the clientconn about the GOAWAY before we set the state to // draining, to allow the client to stop attempting to create streams // before disallowing new streams on this connection. diff --git a/vendor/google.golang.org/grpc/internal/transport/http2_server.go b/vendor/google.golang.org/grpc/internal/transport/http2_server.go index 2b0fde334..28bcba0a3 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http2_server.go +++ b/vendor/google.golang.org/grpc/internal/transport/http2_server.go @@ -945,15 +945,16 @@ func (t *http2Server) streamContextErr(s *Stream) error { // WriteHeader sends the header metadata md back to the client. func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { - if s.updateHeaderSent() { - return ErrIllegalHeaderWrite - } - + s.hdrMu.Lock() + defer s.hdrMu.Unlock() if s.getState() == streamDone { return t.streamContextErr(s) } - s.hdrMu.Lock() + if s.updateHeaderSent() { + return ErrIllegalHeaderWrite + } + if md.Len() > 0 { if s.header.Len() > 0 { s.header = metadata.Join(s.header, md) @@ -962,10 +963,8 @@ func (t *http2Server) WriteHeader(s *Stream, md metadata.MD) error { } } if err := t.writeHeaderLocked(s); err != nil { - s.hdrMu.Unlock() return status.Convert(err).Err() } - s.hdrMu.Unlock() return nil } @@ -1013,17 +1012,19 @@ func (t *http2Server) writeHeaderLocked(s *Stream) error { // TODO(zhaoq): Now it indicates the end of entire stream. Revisit if early // OK is adopted. func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { + s.hdrMu.Lock() + defer s.hdrMu.Unlock() + if s.getState() == streamDone { return nil } - s.hdrMu.Lock() + // TODO(mmukhi): Benchmark if the performance gets better if count the metadata and other header fields // first and create a slice of that exact size. headerFields := make([]hpack.HeaderField, 0, 2) // grpc-status and grpc-message will be there if none else. if !s.updateHeaderSent() { // No headers have been sent. if len(s.header) > 0 { // Send a separate header frame. if err := t.writeHeaderLocked(s); err != nil { - s.hdrMu.Unlock() return err } } else { // Send a trailer only response. @@ -1052,7 +1053,7 @@ func (t *http2Server) WriteStatus(s *Stream, st *status.Status) error { endStream: true, onWrite: t.setResetPingStrikes, } - s.hdrMu.Unlock() + success, err := t.controlBuf.execute(t.checkForHeaderListSize, trailingHeader) if !success { if err != nil { diff --git a/vendor/google.golang.org/grpc/internal/transport/http_util.go b/vendor/google.golang.org/grpc/internal/transport/http_util.go index b77513068..56e95788d 100644 --- a/vendor/google.golang.org/grpc/internal/transport/http_util.go +++ b/vendor/google.golang.org/grpc/internal/transport/http_util.go @@ -47,12 +47,6 @@ const ( http2MaxFrameLen = 16384 // 16KB frame // http://http2.github.io/http2-spec/#SettingValues http2InitHeaderTableSize = 4096 - // baseContentType is the base content-type for gRPC. This is a valid - // content-type on it's own, but can also include a content-subtype such as - // "proto" as a suffix after "+" or ";". See - // https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests - // for more details. - ) var ( diff --git a/vendor/google.golang.org/grpc/server.go b/vendor/google.golang.org/grpc/server.go index b54f5bb57..2ad9da7bf 100644 --- a/vendor/google.golang.org/grpc/server.go +++ b/vendor/google.golang.org/grpc/server.go @@ -190,7 +190,7 @@ type ServerOption interface { // EmptyServerOption does not alter the server configuration. It can be embedded // in another structure to build custom server options. // -// Experimental +// # Experimental // // Notice: This type is EXPERIMENTAL and may be changed or removed in a // later release. @@ -305,7 +305,7 @@ func CustomCodec(codec Codec) ServerOption { // https://github.com/grpc/grpc-go/blob/master/Documentation/encoding.md#using-a-codec. // Will be supported throughout 1.x. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -426,7 +426,7 @@ func ChainStreamInterceptor(interceptors ...StreamServerInterceptor) ServerOptio // InTapHandle returns a ServerOption that sets the tap handle for all the server // transport to be created. Only one can be installed. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -442,6 +442,12 @@ func InTapHandle(h tap.ServerInHandle) ServerOption { // StatsHandler returns a ServerOption that sets the stats handler for the server. func StatsHandler(h stats.Handler) ServerOption { return newFuncServerOption(func(o *serverOptions) { + if h == nil { + logger.Error("ignoring nil parameter in grpc.StatsHandler ServerOption") + // Do not allow a nil stats handler, which would otherwise cause + // panics. + return + } o.statsHandlers = append(o.statsHandlers, h) }) } @@ -469,7 +475,7 @@ func UnknownServiceHandler(streamHandler StreamHandler) ServerOption { // new connections. If this is not set, the default is 120 seconds. A zero or // negative value will result in an immediate timeout. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -490,7 +496,7 @@ func MaxHeaderListSize(s uint32) ServerOption { // HeaderTableSize returns a ServerOption that sets the size of dynamic // header table for stream. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -505,7 +511,7 @@ func HeaderTableSize(s uint32) ServerOption { // zero (default) will disable workers and spawn a new goroutine for each // stream. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -898,7 +904,7 @@ func (s *Server) newHTTP2Transport(c net.Conn) transport.ServerTransport { if err != credentials.ErrConnDispatched { // Don't log on ErrConnDispatched and io.EOF to prevent log spam. if err != io.EOF { - channelz.Warning(logger, s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err) + channelz.Info(logger, s.channelzID, "grpc: Server.Serve failed to create ServerTransport: ", err) } c.Close() } @@ -956,19 +962,19 @@ var _ http.Handler = (*Server)(nil) // To share one port (such as 443 for https) between gRPC and an // existing http.Handler, use a root http.Handler such as: // -// if r.ProtoMajor == 2 && strings.HasPrefix( -// r.Header.Get("Content-Type"), "application/grpc") { -// grpcServer.ServeHTTP(w, r) -// } else { -// yourMux.ServeHTTP(w, r) -// } +// if r.ProtoMajor == 2 && strings.HasPrefix( +// r.Header.Get("Content-Type"), "application/grpc") { +// grpcServer.ServeHTTP(w, r) +// } else { +// yourMux.ServeHTTP(w, r) +// } // // Note that ServeHTTP uses Go's HTTP/2 server implementation which is totally // separate from grpc-go's HTTP/2 server. Performance and features may vary // between the two paths. ServeHTTP does not support some gRPC features // available through grpc-go's HTTP/2 server. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -1674,7 +1680,7 @@ type streamKey struct{} // NewContextWithServerTransportStream creates a new context from ctx and // attaches stream to it. // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -1689,7 +1695,7 @@ func NewContextWithServerTransportStream(ctx context.Context, stream ServerTrans // // See also NewContextWithServerTransportStream. // -// Experimental +// # Experimental // // Notice: This type is EXPERIMENTAL and may be changed or removed in a // later release. @@ -1704,7 +1710,7 @@ type ServerTransportStream interface { // ctx. Returns nil if the given context has no stream associated with it // (which implies it is not an RPC invocation context). // -// Experimental +// # Experimental // // Notice: This API is EXPERIMENTAL and may be changed or removed in a // later release. @@ -1825,12 +1831,12 @@ func (s *Server) getCodec(contentSubtype string) baseCodec { // When called multiple times, all the provided metadata will be merged. All // the metadata will be sent out when one of the following happens: // -// - grpc.SendHeader is called, or for streaming handlers, stream.SendHeader. -// - The first response message is sent. For unary handlers, this occurs when -// the handler returns; for streaming handlers, this can happen when stream's -// SendMsg method is called. -// - An RPC status is sent out (error or success). This occurs when the handler -// returns. +// - grpc.SendHeader is called, or for streaming handlers, stream.SendHeader. +// - The first response message is sent. For unary handlers, this occurs when +// the handler returns; for streaming handlers, this can happen when stream's +// SendMsg method is called. +// - An RPC status is sent out (error or success). This occurs when the handler +// returns. // // SetHeader will fail if called after any of the events above. // diff --git a/vendor/google.golang.org/grpc/service_config.go b/vendor/google.golang.org/grpc/service_config.go index b01c548bb..01bbb2025 100644 --- a/vendor/google.golang.org/grpc/service_config.go +++ b/vendor/google.golang.org/grpc/service_config.go @@ -57,10 +57,9 @@ type lbConfig struct { type ServiceConfig struct { serviceconfig.Config - // LB is the load balancer the service providers recommends. The balancer - // specified via grpc.WithBalancerName will override this. This is deprecated; - // lbConfigs is preferred. If lbConfig and LB are both present, lbConfig - // will be used. + // LB is the load balancer the service providers recommends. This is + // deprecated; lbConfigs is preferred. If lbConfig and LB are both present, + // lbConfig will be used. LB *string // lbConfig is the service config's load balancing configuration. If diff --git a/vendor/google.golang.org/grpc/stream.go b/vendor/google.golang.org/grpc/stream.go index 6d82e0d7c..446a91e32 100644 --- a/vendor/google.golang.org/grpc/stream.go +++ b/vendor/google.golang.org/grpc/stream.go @@ -140,13 +140,13 @@ type ClientStream interface { // To ensure resources are not leaked due to the stream returned, one of the following // actions must be performed: // -// 1. Call Close on the ClientConn. -// 2. Cancel the context provided. -// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated -// client-streaming RPC, for instance, might use the helper function -// CloseAndRecv (note that CloseSend does not Recv, therefore is not -// guaranteed to release all resources). -// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. +// 1. Call Close on the ClientConn. +// 2. Cancel the context provided. +// 3. Call RecvMsg until a non-nil error is returned. A protobuf-generated +// client-streaming RPC, for instance, might use the helper function +// CloseAndRecv (note that CloseSend does not Recv, therefore is not +// guaranteed to release all resources). +// 4. Receive a non-nil, non-io.EOF error from Header or SendMsg. // // If none of the above happen, a goroutine and a context will be leaked, and grpc // will not call the optionally-configured stats handler with a stats.End message. @@ -303,12 +303,6 @@ func newClientStreamWithParams(ctx context.Context, desc *StreamDesc, cc *Client } cs.binlog = binarylog.GetMethodLogger(method) - cs.attempt, err = cs.newAttemptLocked(false /* isTransparent */) - if err != nil { - cs.finish(err) - return nil, err - } - // Pick the transport to use and create a new stream on the transport. // Assign cs.attempt upon success. op := func(a *csAttempt) error { @@ -704,6 +698,18 @@ func (cs *clientStream) withRetry(op func(a *csAttempt) error, onSuccess func()) // already be status errors. return toRPCErr(op(cs.attempt)) } + if len(cs.buffer) == 0 { + // For the first op, which controls creation of the stream and + // assigns cs.attempt, we need to create a new attempt inline + // before executing the first op. On subsequent ops, the attempt + // is created immediately before replaying the ops. + var err error + if cs.attempt, err = cs.newAttemptLocked(false /* isTransparent */); err != nil { + cs.mu.Unlock() + cs.finish(err) + return err + } + } a := cs.attempt cs.mu.Unlock() err := op(a) diff --git a/vendor/google.golang.org/grpc/version.go b/vendor/google.golang.org/grpc/version.go index 0eb2998cb..8934f06bc 100644 --- a/vendor/google.golang.org/grpc/version.go +++ b/vendor/google.golang.org/grpc/version.go @@ -19,4 +19,4 @@ package grpc // Version is the current grpc version. -const Version = "1.48.0" +const Version = "1.49.0" diff --git a/vendor/google.golang.org/grpc/vet.sh b/vendor/google.golang.org/grpc/vet.sh index ceb436c6c..c3fc8253b 100644 --- a/vendor/google.golang.org/grpc/vet.sh +++ b/vendor/google.golang.org/grpc/vet.sh @@ -147,7 +147,6 @@ grpc.NewGZIPDecompressor grpc.RPCCompressor grpc.RPCDecompressor grpc.ServiceConfig -grpc.WithBalancerName grpc.WithCompressor grpc.WithDecompressor grpc.WithDialer diff --git a/vendor/modules.txt b/vendor/modules.txt index c2e53d7a9..5bfe3d48c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -679,8 +679,8 @@ google.golang.org/appengine/urlfetch google.golang.org/genproto/googleapis/api/httpbody google.golang.org/genproto/googleapis/rpc/status google.golang.org/genproto/protobuf/field_mask -# google.golang.org/grpc v1.48.0 -## explicit; go 1.14 +# google.golang.org/grpc v1.49.0 +## explicit; go 1.17 google.golang.org/grpc google.golang.org/grpc/attributes google.golang.org/grpc/backoff From 71dbc7dbb45d771e4885efef8535708c76f6a453 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 5 Sep 2022 10:41:37 +0530 Subject: [PATCH 04/11] rbd: map only primary image If the image is mirroring enabled and primary consider it for mapping, if the image is mirroring enabled but not primary yet. return error message until the image is marked as primary. Signed-off-by: Madhu Rajanna --- internal/rbd/rbd_util.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 398557318..d32c4d391 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -138,8 +138,6 @@ type rbdImage struct { // an opened IOContext, call .openIoctx() before using ioctx *rados.IOContext - // Primary represent if the image is primary or not. - Primary bool // Set metadata on volume EnableMetadata bool } @@ -512,6 +510,10 @@ func (ri *rbdImage) open() (*librbd.Image, error) { // isInUse checks if there is a watcher on the image. It returns true if there // is a watcher on the image, otherwise returns false. +// In case of mirroring, the image should be primary to check watchers if the +// image is secondary it returns an error. +// isInUse is called with exponential backoff to check the image is used by +// anyone else the returned bool value is discarded if its a RWX access. func (ri *rbdImage) isInUse() (bool, error) { image, err := ri.open() if err != nil { @@ -532,11 +534,16 @@ func (ri *rbdImage) isInUse() (bool, error) { if err != nil { return false, err } - ri.Primary = mirrorInfo.Primary + + if mirrorInfo.State == librbd.MirrorImageEnabled && !mirrorInfo.Primary { + // Mapping secondary image can cause issues.returning error as the + // bool value is discarded if it its RWX access. + return false, fmt.Errorf("cannot map image %s it is not primary", ri) + } // because we opened the image, there is at least one watcher defaultWatchers := 1 - if ri.Primary { + if mirrorInfo.Primary { // if rbd mirror daemon is running, a watcher will be added by the rbd // mirror daemon for mirrored images. defaultWatchers++ From 9d46478794721f2887904ebe6340ef03c5a8aa45 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 6 Sep 2022 10:36:10 +0530 Subject: [PATCH 05/11] ci: use resync to sync helm charts When a file on source is deleted same need to be deleted on the destination, with rsync we can achieve it. fixes: #3329 Signed-off-by: Madhu Rajanna --- deploy.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 0dc77f57f..0cc245a7a 100755 --- a/deploy.sh +++ b/deploy.sh @@ -77,7 +77,8 @@ push_helm_charts() { fi mkdir -p "${CHARTDIR}/csi-charts/docs/${PACKAGE}" - cp -R "./charts/ceph-csi-${PACKAGE}" "${CHARTDIR}/csi-charts/docs/${PACKAGE}" + # Use rsync to remove files from destination when source file is deleted. + rsync -avh "./charts/ceph-csi-${PACKAGE}" "${CHARTDIR}/csi-charts/docs/${PACKAGE}" --delete pushd "${CHARTDIR}/csi-charts/docs/${PACKAGE}" >/dev/null helm package "ceph-csi-${PACKAGE}" popd >/dev/null From e56621cd669dff43377e53893d8caffdbd1c08ae Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 7 Sep 2022 11:46:22 +0530 Subject: [PATCH 06/11] cephfs: fix subvolumegroup creation for multiple fs In a cluster we can have multiple filesystem for that we need to have a map of subvolumegroups to check filesystem is created nor not. Signed-off-by: Madhu Rajanna --- internal/cephfs/core/metadata.go | 4 +-- internal/cephfs/core/snapshot_metadata.go | 4 +-- internal/cephfs/core/volume.go | 33 ++++++++++++----------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/internal/cephfs/core/metadata.go b/internal/cephfs/core/metadata.go index e0cedbd72..d5fb8a73a 100644 --- a/internal/cephfs/core/metadata.go +++ b/internal/cephfs/core/metadata.go @@ -33,9 +33,7 @@ const ( var ErrSubVolMetadataNotSupported = errors.New("subvolume metadata operations are not supported") func (s *subVolumeClient) supportsSubVolMetadata() bool { - if _, keyPresent := clusterAdditionalInfo[s.clusterID]; !keyPresent { - clusterAdditionalInfo[s.clusterID] = &localClusterState{} - } + newLocalClusterState(s.clusterID) return clusterAdditionalInfo[s.clusterID].subVolMetadataState != unsupported } diff --git a/internal/cephfs/core/snapshot_metadata.go b/internal/cephfs/core/snapshot_metadata.go index 5d00798c7..5cf67ddc0 100644 --- a/internal/cephfs/core/snapshot_metadata.go +++ b/internal/cephfs/core/snapshot_metadata.go @@ -29,9 +29,7 @@ import ( var ErrSubVolSnapMetadataNotSupported = errors.New("subvolume snapshot metadata operations are not supported") func (s *snapshotClient) supportsSubVolSnapMetadata() bool { - if _, keyPresent := clusterAdditionalInfo[s.clusterID]; !keyPresent { - clusterAdditionalInfo[s.clusterID] = &localClusterState{} - } + newLocalClusterState(s.clusterID) return clusterAdditionalInfo[s.clusterID].subVolSnapshotMetadataState != unsupported } diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index 316d2c7cd..ee0611c9e 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -202,18 +202,25 @@ type localClusterState struct { resizeState operationState subVolMetadataState operationState subVolSnapshotMetadataState operationState + // A cluster can have multiple filesystem for that we need to have a map of + // subvolumegroups to check filesystem is created nor not. // set true once a subvolumegroup is created - // for corresponding cluster. - subVolumeGroupCreated bool + // for corresponding filesystem in a cluster. + subVolumeGroupsCreated map[string]bool +} + +func newLocalClusterState(clusterID string) { + // verify if corresponding clusterID key is present in the map, + // and if not, initialize with default values(false). + if _, keyPresent := clusterAdditionalInfo[clusterID]; !keyPresent { + clusterAdditionalInfo[clusterID] = &localClusterState{} + clusterAdditionalInfo[clusterID].subVolumeGroupsCreated = make(map[string]bool) + } } // CreateVolume creates a subvolume. func (s *subVolumeClient) CreateVolume(ctx context.Context) error { - // verify if corresponding clusterID key is present in the map, - // and if not, initialize with default values(false). - if _, keyPresent := clusterAdditionalInfo[s.clusterID]; !keyPresent { - clusterAdditionalInfo[s.clusterID] = &localClusterState{} - } + newLocalClusterState(s.clusterID) ca, err := s.conn.GetFSAdmin() if err != nil { @@ -223,7 +230,7 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { } // create subvolumegroup if not already created for the cluster. - if !clusterAdditionalInfo[s.clusterID].subVolumeGroupCreated { + if !clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] { opts := fsAdmin.SubVolumeGroupOptions{} err = ca.CreateSubVolumeGroup(s.FsName, s.SubvolumeGroup, &opts) if err != nil { @@ -237,7 +244,7 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { return err } log.DebugLog(ctx, "cephfs: created subvolume group %s", s.SubvolumeGroup) - clusterAdditionalInfo[s.clusterID].subVolumeGroupCreated = true + clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] = true } opts := fsAdmin.SubVolumeOptions{ @@ -279,13 +286,7 @@ func (s *subVolumeClient) ExpandVolume(ctx context.Context, bytesQuota int64) er // subvolume. If the command is not available as a fallback it will use // CreateVolume to resize the subvolume. func (s *subVolumeClient) ResizeVolume(ctx context.Context, bytesQuota int64) error { - // keyPresent checks whether corresponding clusterID key is present in clusterAdditionalInfo - var keyPresent bool - // verify if corresponding clusterID key is present in the map, - // and if not, initialize with default values(false). - if _, keyPresent = clusterAdditionalInfo[s.clusterID]; !keyPresent { - clusterAdditionalInfo[s.clusterID] = &localClusterState{} - } + newLocalClusterState(s.clusterID) // resize subvolume when either it's supported, or when corresponding // clusterID key was not present. if clusterAdditionalInfo[s.clusterID].resizeState == unknown || From 76064d8e34eb40bc9a4f66fd65021052cdaccb35 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 7 Sep 2022 11:50:45 +0530 Subject: [PATCH 07/11] cephfs: retry subvolumegroup creation Incase the subvolumegroup is deleted and recreated we need to restart the cephcsi provisioner pod to clear cache that cephcsi maintains. With this PR if cephcsi sees NotFound error duing subvolume creation it will reset the cache for that filesystem so that in next RPC call cephcsi will try to create the subvolumegroup again Ref: https://github.com/rook/rook/issues/10623 Signed-off-by: Madhu Rajanna --- internal/cephfs/core/volume.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index ee0611c9e..ff71baf41 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -259,6 +259,12 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { if err != nil { log.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", s.VolID, s.FsName, err) + if errors.Is(err, rados.ErrNotFound) { + // Reset the subVolumeGroupsCreated so that we can try again to create the + // subvolumegroup in next request if the error is Not Found. + clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] = false + } + return err } From 2a6487cbf5a71e58aa95baf594dc621bda4fbb8e Mon Sep 17 00:00:00 2001 From: Prashanth Dintyala Date: Wed, 7 Sep 2022 13:49:38 -0700 Subject: [PATCH 08/11] rbd: create token and use it for vault SA everytime possible use TokenRequest API by default for vault SA even with K8s versions < 1.24 Signed-off-by: Prashanth Dintyala --- internal/kms/vault_sa.go | 50 ++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/internal/kms/vault_sa.go b/internal/kms/vault_sa.go index 4a65768dd..e06b93e5e 100644 --- a/internal/kms/vault_sa.go +++ b/internal/kms/vault_sa.go @@ -23,8 +23,6 @@ import ( "io/ioutil" "os" - "github.com/ceph/ceph-csi/internal/util/k8s" - "github.com/libopenstorage/secrets/vault" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" @@ -39,9 +37,6 @@ const ( // should be available in the Tenants namespace. This ServiceAccount // will be used to connect to Hashicorp Vault. vaultTenantSAName = "ceph-csi-vault-sa" - // Kubernetes version which requires ServiceAccount token creation. - // Kubernetes 1.24 => 1 * 1000 + 24. - kubeMinVersionForCreateToken = 1024 ) /* @@ -298,6 +293,15 @@ func (kms *vaultTenantSA) getToken() (string, error) { kms.tenantSAName, err) } + // From Kubernetes v1.24+, secret for service account tokens are not + // automatically created. Trying to fetch tokens from service account secret references will fail + // refer: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md \ + // #no-really-you-must-read-this-before-you-upgrade-1. + token, err := kms.createToken(sa, c) + if err == nil { + return token, nil + } + for _, secretRef := range sa.Secrets { secret, sErr := c.CoreV1().Secrets(kms.Tenant).Get(context.TODO(), secretRef.Name, metav1.GetOptions{}) if sErr != nil { @@ -310,7 +314,7 @@ func (kms *vaultTenantSA) getToken() (string, error) { } } - return kms.createToken(sa, c) + return "", fmt.Errorf("failed to find/create ServiceAccount token %s/%s: %w", kms.Tenant, kms.tenantSAName, err) } // getTokenPath creates a temporary directory structure that contains the token @@ -335,32 +339,18 @@ func (kms *vaultTenantSA) getTokenPath() (string, error) { return dir + "/token", nil } -// createToken creates required service account token for kubernetes 1.24+, -// else returns error. -// From kubernetes v1.24+, secret for service account tokens are not -// automatically created. Hence, use the create token api to fetch it. -// refer: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md \ -// #no-really-you-must-read-this-before-you-upgrade-1 . +// createToken creates required service account token using the TokenRequest API. func (kms *vaultTenantSA) createToken(sa *corev1.ServiceAccount, client *kubernetes.Clientset) (string, error) { - major, minor, err := k8s.GetServerVersion(client) + tokenRequest := &authenticationv1.TokenRequest{} + token, err := client.CoreV1().ServiceAccounts(kms.Tenant).CreateToken( + context.TODO(), + sa.Name, + tokenRequest, + metav1.CreateOptions{}, + ) if err != nil { - return "", fmt.Errorf("failed to get server version: %w", err) + return "", fmt.Errorf("failed to create token for service account %s/%s: %w", kms.Tenant, sa.Name, err) } - if (major*1000 + minor) >= kubeMinVersionForCreateToken { - tokenRequest := &authenticationv1.TokenRequest{} - token, err := client.CoreV1().ServiceAccounts(kms.Tenant).CreateToken( - context.TODO(), - sa.Name, - tokenRequest, - metav1.CreateOptions{}, - ) - if err != nil { - return "", fmt.Errorf("failed to create token for service account %s/%s: %w", kms.Tenant, sa.Name, err) - } - - return token.Status.Token, nil - } - - return "", fmt.Errorf("failed to find token in ServiceAccount %s/%s", kms.Tenant, kms.tenantSAName) + return token.Status.Token, nil } From 3d04065d005baf4b1d92efe0f18e9bb0db160430 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Sep 2022 20:21:31 +0000 Subject: [PATCH 09/11] rebase: bump k8s.io/klog/v2 from 2.70.1 to 2.80.1 Bumps [k8s.io/klog/v2](https://github.com/kubernetes/klog) from 2.70.1 to 2.80.1. - [Release notes](https://github.com/kubernetes/klog/releases) - [Changelog](https://github.com/kubernetes/klog/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes/klog/compare/v2.70.1...v2.80.1) --- updated-dependencies: - dependency-name: k8s.io/klog/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 3 +- vendor/k8s.io/klog/v2/OWNERS | 1 + vendor/k8s.io/klog/v2/contextual.go | 5 +- .../klog/v2/internal/serialize/keyvalues.go | 2 +- vendor/k8s.io/klog/v2/klog.go | 137 +++++++++--------- vendor/modules.txt | 2 +- 7 files changed, 79 insertions(+), 73 deletions(-) diff --git a/go.mod b/go.mod index f34bfb2dc..b8d371fc4 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( k8s.io/apimachinery v0.25.0 k8s.io/client-go v12.0.0+incompatible k8s.io/cloud-provider v0.25.0 - k8s.io/klog/v2 v2.70.1 + k8s.io/klog/v2 v2.80.1 // // when updating k8s.io/kubernetes, make sure to update the replace section too // diff --git a/go.sum b/go.sum index eb7990c36..9a201067d 100644 --- a/go.sum +++ b/go.sum @@ -1867,8 +1867,9 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.25.0/go.mod h1:dfdl4aQkleiWK/U++UDLdDC8g2rsonhkB23zzUeBCgM= k8s.io/kube-controller-manager v0.25.0/go.mod h1:SjL1hKSG2z9wajnvjRHZv1zOsdDHjmbZd1ykmaYO6J8= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= diff --git a/vendor/k8s.io/klog/v2/OWNERS b/vendor/k8s.io/klog/v2/OWNERS index 8cccebf2e..a2fe8f351 100644 --- a/vendor/k8s.io/klog/v2/OWNERS +++ b/vendor/k8s.io/klog/v2/OWNERS @@ -1,5 +1,6 @@ # See the OWNERS docs at https://go.k8s.io/owners reviewers: + - harshanarayana - pohly approvers: - dims diff --git a/vendor/k8s.io/klog/v2/contextual.go b/vendor/k8s.io/klog/v2/contextual.go index 65ac479ab..2428963c0 100644 --- a/vendor/k8s.io/klog/v2/contextual.go +++ b/vendor/k8s.io/klog/v2/contextual.go @@ -47,8 +47,9 @@ var ( // If set, all log lines will be suppressed from the regular output, and // redirected to the logr implementation. // Use as: -// ... -// klog.SetLogger(zapr.NewLogger(zapLog)) +// +// ... +// klog.SetLogger(zapr.NewLogger(zapLog)) // // To remove a backing logr implemention, use ClearLogger. Setting an // empty logger with SetLogger(logr.Logger{}) does not work. diff --git a/vendor/k8s.io/klog/v2/internal/serialize/keyvalues.go b/vendor/k8s.io/klog/v2/internal/serialize/keyvalues.go index f85d7ccf8..ad6bf1116 100644 --- a/vendor/k8s.io/klog/v2/internal/serialize/keyvalues.go +++ b/vendor/k8s.io/klog/v2/internal/serialize/keyvalues.go @@ -145,7 +145,7 @@ func KVListFormat(b *bytes.Buffer, keysAndValues ...interface{}) { case string: writeStringValue(b, true, value) default: - writeStringValue(b, false, fmt.Sprintf("%+v", v)) + writeStringValue(b, false, fmt.Sprintf("%+v", value)) } case []byte: // In https://github.com/kubernetes/klog/pull/237 it was decided diff --git a/vendor/k8s.io/klog/v2/klog.go b/vendor/k8s.io/klog/v2/klog.go index 652fadcd4..1bd11b675 100644 --- a/vendor/k8s.io/klog/v2/klog.go +++ b/vendor/k8s.io/klog/v2/klog.go @@ -39,39 +39,38 @@ // This package provides several flags that modify this behavior. // As a result, flag.Parse must be called before any logging is done. // -// -logtostderr=true -// Logs are written to standard error instead of to files. -// This shortcuts most of the usual output routing: -// -alsologtostderr, -stderrthreshold and -log_dir have no -// effect and output redirection at runtime with SetOutput is -// ignored. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. +// -logtostderr=true +// Logs are written to standard error instead of to files. +// This shortcuts most of the usual output routing: +// -alsologtostderr, -stderrthreshold and -log_dir have no +// effect and output redirection at runtime with SetOutput is +// ignored. +// -alsologtostderr=false +// Logs are written to standard error as well as to files. +// -stderrthreshold=ERROR +// Log events at or above this severity are logged to standard +// error as well as to files. +// -log_dir="" +// Log files will be written to this directory instead of the +// default temporary directory. // -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". +// Other flags provide aids to debugging. // +// -log_backtrace_at="" +// When set to a file and line number holding a logging statement, +// such as +// -log_backtrace_at=gopherflakes.go:234 +// a stack trace will be written to the Info log whenever execution +// hits that statement. (Unlike with -vmodule, the ".go" must be +// present.) +// -v=0 +// Enable V-leveled logging at the specified level. +// -vmodule="" +// The syntax of the argument is a comma-separated list of pattern=N, +// where pattern is a literal file name (minus the ".go" suffix) or +// "glob" pattern and N is a V level. For instance, +// -vmodule=gopher*=3 +// sets the V level to 3 in all Go files whose names begin "gopher". package klog import ( @@ -397,45 +396,48 @@ type flushSyncWriter interface { io.Writer } -// init sets up the defaults. +var logging loggingT +var commandLine flag.FlagSet + +// init sets up the defaults and creates command line flags. func init() { + commandLine.StringVar(&logging.logDir, "log_dir", "", "If non-empty, write log files in this directory (no effect when -logtostderr=true)") + commandLine.StringVar(&logging.logFile, "log_file", "", "If non-empty, use this log file (no effect when -logtostderr=true)") + commandLine.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", 1800, + "Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. "+ + "If the value is 0, the maximum file size is unlimited.") + commandLine.BoolVar(&logging.toStderr, "logtostderr", true, "log to standard error instead of files") + commandLine.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files (no effect when -logtostderr=true)") + logging.setVState(0, nil, false) + commandLine.Var(&logging.verbosity, "v", "number for the log level verbosity") + commandLine.BoolVar(&logging.addDirHeader, "add_dir_header", false, "If true, adds the file directory to the header of the log messages") + commandLine.BoolVar(&logging.skipHeaders, "skip_headers", false, "If true, avoid header prefixes in the log messages") + commandLine.BoolVar(&logging.oneOutput, "one_output", false, "If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)") + commandLine.BoolVar(&logging.skipLogHeaders, "skip_log_headers", false, "If true, avoid headers when opening log files (no effect when -logtostderr=true)") logging.stderrThreshold = severityValue{ Severity: severity.ErrorLog, // Default stderrThreshold is ERROR. } - logging.setVState(0, nil, false) - logging.logDir = "" - logging.logFile = "" - logging.logFileMaxSizeMB = 1800 - logging.toStderr = true - logging.alsoToStderr = false - logging.skipHeaders = false - logging.addDirHeader = false - logging.skipLogHeaders = false - logging.oneOutput = false + commandLine.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=false)") + commandLine.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") + commandLine.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") + + logging.settings.contextualLoggingEnabled = true logging.flushD = newFlushDaemon(logging.lockAndFlushAll, nil) } // InitFlags is for explicitly initializing the flags. +// It may get called repeatedly for different flagsets, but not +// twice for the same one. May get called concurrently +// to other goroutines using klog. However, only some flags +// may get set concurrently (see implementation). func InitFlags(flagset *flag.FlagSet) { if flagset == nil { flagset = flag.CommandLine } - flagset.StringVar(&logging.logDir, "log_dir", logging.logDir, "If non-empty, write log files in this directory (no effect when -logtostderr=true)") - flagset.StringVar(&logging.logFile, "log_file", logging.logFile, "If non-empty, use this log file (no effect when -logtostderr=true)") - flagset.Uint64Var(&logging.logFileMaxSizeMB, "log_file_max_size", logging.logFileMaxSizeMB, - "Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. "+ - "If the value is 0, the maximum file size is unlimited.") - flagset.BoolVar(&logging.toStderr, "logtostderr", logging.toStderr, "log to standard error instead of files") - flagset.BoolVar(&logging.alsoToStderr, "alsologtostderr", logging.alsoToStderr, "log to standard error as well as files (no effect when -logtostderr=true)") - flagset.Var(&logging.verbosity, "v", "number for the log level verbosity") - flagset.BoolVar(&logging.addDirHeader, "add_dir_header", logging.addDirHeader, "If true, adds the file directory to the header of the log messages") - flagset.BoolVar(&logging.skipHeaders, "skip_headers", logging.skipHeaders, "If true, avoid header prefixes in the log messages") - flagset.BoolVar(&logging.oneOutput, "one_output", logging.oneOutput, "If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)") - flagset.BoolVar(&logging.skipLogHeaders, "skip_log_headers", logging.skipLogHeaders, "If true, avoid headers when opening log files (no effect when -logtostderr=true)") - flagset.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=false)") - flagset.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flagset.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") + commandLine.VisitAll(func(f *flag.Flag) { + flagset.Var(f.Value, f.Name, f.Usage) + }) } // Flush flushes all pending log I/O. @@ -550,12 +552,6 @@ type loggingT struct { vmap map[uintptr]Level } -var logging = loggingT{ - settings: settings{ - contextualLoggingEnabled: true, - }, -} - // setVState sets a consistent state for V logging. // l.mu is held. func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { @@ -633,8 +629,11 @@ It returns a buffer containing the formatted header and the user's file and line The depth specifies how many stack frames above lives the source line to be identified in the log message. Log lines have this form: + Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... + where the fields are defined as follows: + L A single character, representing the log level (eg 'I' for INFO) mm The month (zero padded; ie May is '05') dd The day (zero padded) @@ -1298,9 +1297,13 @@ func newVerbose(level Level, b bool) Verbose { // The returned value is a struct of type Verbose, which implements Info, Infoln // and Infof. These methods will write to the Info log if called. // Thus, one may write either +// // if klog.V(2).Enabled() { klog.Info("log this") } +// // or +// // klog.V(2).Info("log this") +// // The second form is shorter but the first is cheaper if logging is off because it does // not evaluate its arguments. // @@ -1582,10 +1585,10 @@ func ErrorSDepth(depth int, err error, msg string, keysAndValues ...interface{}) // // Callers who want more control over handling of fatal events may instead use a // combination of different functions: -// - some info or error logging function, optionally with a stack trace -// value generated by github.com/go-logr/lib/dbg.Backtrace -// - Flush to flush pending log data -// - panic, os.Exit or returning to the caller with an error +// - some info or error logging function, optionally with a stack trace +// value generated by github.com/go-logr/lib/dbg.Backtrace +// - Flush to flush pending log data +// - panic, os.Exit or returning to the caller with an error // // Arguments are handled in the manner of fmt.Print; a newline is appended if missing. func Fatal(args ...interface{}) { diff --git a/vendor/modules.txt b/vendor/modules.txt index 5bfe3d48c..5784e1436 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1207,7 +1207,7 @@ k8s.io/component-helpers/node/util/sysctl k8s.io/component-helpers/scheduling/corev1 k8s.io/component-helpers/scheduling/corev1/nodeaffinity k8s.io/component-helpers/storage/volume -# k8s.io/klog/v2 v2.70.1 +# k8s.io/klog/v2 v2.80.1 ## explicit; go 1.13 k8s.io/klog/v2 k8s.io/klog/v2/internal/buffer From a57859dfa404dbef8f4ba55df0ae88158c897d5c Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Tue, 13 Sep 2022 15:43:21 +0530 Subject: [PATCH 10/11] rbd: use blocklist range cmd, fallback if it fails This commit adds blocklist range cmd feature, while fallbacks to old blocklist one ip at a time if the cmd is invalid(not available). Signed-off-by: Rakshith R --- internal/csi-addons/networkfence/fencing.go | 62 ++++++++++++++++++--- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/internal/csi-addons/networkfence/fencing.go b/internal/csi-addons/networkfence/fencing.go index 8d40ff6b9..713b2a10e 100644 --- a/internal/csi-addons/networkfence/fencing.go +++ b/internal/csi-addons/networkfence/fencing.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net" + "strings" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -25,7 +26,10 @@ import ( "github.com/csi-addons/spec/lib/go/fence" ) -const blocklistTime = "157784760" +const ( + blocklistTime = "157784760" + invalidCommandStr = "invalid command" +) // NetworkFence contains the CIDR blocks to be blocked. type NetworkFence struct { @@ -65,7 +69,7 @@ func NewNetworkFence( } // addCephBlocklist adds an IP to ceph osd blocklist. -func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { +func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string, useRange bool) error { arg := []string{ "--id", nf.cr.ID, "--keyfile=" + nf.cr.KeyFile, @@ -77,11 +81,18 @@ func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { // represent infinity from ceph-csi side. // At any point in this time, the IPs can be unblocked by an UnfenceClusterReq. // This needs to be updated once ceph provides functionality for the same. - cmd := []string{"osd", "blocklist", "add", ip, blocklistTime} + cmd := []string{"osd", "blocklist"} + if useRange { + cmd = append(cmd, "range") + } + cmd = append(cmd, "add", ip, blocklistTime) cmd = append(cmd, arg...) - _, _, err := util.ExecCommand(ctx, "ceph", cmd...) + _, stdErr, err := util.ExecCommand(ctx, "ceph", cmd...) if err != nil { - return fmt.Errorf("failed to blocklist IP %q: %w", ip, err) + return fmt.Errorf("failed to blocklist IP %q: %w stderr: %q", ip, err, stdErr) + } + if stdErr != "" { + return fmt.Errorf("failed to blocklist IP %q: %q", ip, stdErr) } log.DebugLog(ctx, "blocklisted IP %q successfully", ip) @@ -91,8 +102,21 @@ func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { // AddNetworkFence blocks access for all the IPs in the IP range mentioned via the CIDR block // using a network fence. func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { + hasBlocklistRangeSupport := true // for each CIDR block, convert it into a range of IPs so as to perform blocklisting operation. for _, cidr := range nf.Cidr { + // try range blocklist cmd, if invalid fallback to + // iterating through IP range. + if hasBlocklistRangeSupport { + err := nf.addCephBlocklist(ctx, cidr, true) + if err == nil { + continue + } + if !strings.Contains(err.Error(), invalidCommandStr) { + return fmt.Errorf("failed to add blocklist range %q: %w", cidr, err) + } + hasBlocklistRangeSupport = false + } // fetch the list of IPs from a CIDR block hosts, err := getIPRange(cidr) if err != nil { @@ -101,7 +125,7 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { // add ceph blocklist for each IP in the range mentioned by the CIDR for _, host := range hosts { - err = nf.addCephBlocklist(ctx, host) + err = nf.addCephBlocklist(ctx, host, false) if err != nil { return err } @@ -154,19 +178,26 @@ func GetCIDR(cidrs Cidrs) ([]string, error) { } // removeCephBlocklist removes an IP from ceph osd blocklist. -func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string) error { +func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string, useRange bool) error { arg := []string{ "--id", nf.cr.ID, "--keyfile=" + nf.cr.KeyFile, "-m", nf.Monitors, } - cmd := []string{"osd", "blocklist", "rm", ip} + cmd := []string{"osd", "blocklist"} + if useRange { + cmd = append(cmd, "range") + } + cmd = append(cmd, "rm", ip) cmd = append(cmd, arg...) _, stdErr, err := util.ExecCommand(ctx, "ceph", cmd...) if err != nil { return fmt.Errorf("failed to unblock IP %q: %v %w", ip, stdErr, err) } + if stdErr != "" { + return fmt.Errorf("failed to unblock IP %q: %q", ip, stdErr) + } log.DebugLog(ctx, "unblocked IP %q successfully", ip) return nil @@ -175,8 +206,21 @@ func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string) erro // RemoveNetworkFence unblocks access for all the IPs in the IP range mentioned via the CIDR block // using a network fence. func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error { + hasBlocklistRangeSupport := true // for each CIDR block, convert it into a range of IPs so as to undo blocklisting operation. for _, cidr := range nf.Cidr { + // try range blocklist cmd, if invalid fallback to + // iterating through IP range. + if hasBlocklistRangeSupport { + err := nf.removeCephBlocklist(ctx, cidr, true) + if err == nil { + continue + } + if !strings.Contains(err.Error(), invalidCommandStr) { + return fmt.Errorf("failed to remove blocklist range %q: %w", cidr, err) + } + hasBlocklistRangeSupport = false + } // fetch the list of IPs from a CIDR block hosts, err := getIPRange(cidr) if err != nil { @@ -184,7 +228,7 @@ func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error { } // remove ceph blocklist for each IP in the range mentioned by the CIDR for _, host := range hosts { - err := nf.removeCephBlocklist(ctx, host) + err := nf.removeCephBlocklist(ctx, host, false) if err != nil { return err } From b19705f26045fcf4f911870974ceaff1ae8ab997 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Fri, 9 Sep 2022 16:56:41 +0530 Subject: [PATCH 11/11] rbd: implements getVolumeReplicationInfo This commit implements getVolumeReplicationInfo to get the last sync time and update it in volume replication CR. Signed-off-by: yati1998 --- go.mod | 2 +- go.sum | 2 + internal/rbd/replicationcontrollerserver.go | 112 ++++ .../rbd/replicationcontrollerserver_test.go | 54 ++ .../spec/lib/go/replication/replication.pb.go | 591 ++++++++++++------ .../lib/go/replication/replication_grpc.pb.go | 40 ++ vendor/modules.txt | 2 +- 7 files changed, 603 insertions(+), 200 deletions(-) diff --git a/go.mod b/go.mod index b8d371fc4..71256458c 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/ceph/go-ceph v0.17.0 github.com/container-storage-interface/spec v1.6.0 github.com/csi-addons/replication-lib-utils v0.2.0 - github.com/csi-addons/spec v0.1.2-0.20220829042231-b27a0d84b50b + github.com/csi-addons/spec v0.1.2-0.20220906123848-52ce69f90900 github.com/gemalto/kmip-go v0.0.8-0.20220721195433-3fe83e2d3f26 github.com/golang/protobuf v1.5.2 github.com/google/uuid v1.3.0 diff --git a/go.sum b/go.sum index 9a201067d..ac5572bd5 100644 --- a/go.sum +++ b/go.sum @@ -280,6 +280,8 @@ github.com/csi-addons/replication-lib-utils v0.2.0/go.mod h1:ROQlEsc2EerVtc/K/C+ github.com/csi-addons/spec v0.1.0/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= github.com/csi-addons/spec v0.1.2-0.20220829042231-b27a0d84b50b h1:C5KgryC4RwQVSF8L/pgcKftgn7Z1zHFZlACJukPlCxs= github.com/csi-addons/spec v0.1.2-0.20220829042231-b27a0d84b50b/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= +github.com/csi-addons/spec v0.1.2-0.20220906123848-52ce69f90900 h1:zX0138DipZsZqxK1UwAmaRZmL89OuQMkwh7FtvTDgFw= +github.com/csi-addons/spec v0.1.2-0.20220906123848-52ce69f90900/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI= github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index b5b132392..e7d359a55 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -18,6 +18,7 @@ package rbd import ( "context" + "encoding/json" "errors" "fmt" "regexp" @@ -35,6 +36,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" ) // imageMirroringMode is used to indicate the mirroring mode for an RBD image. @@ -834,6 +836,116 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, return resp, nil } +// GetVolumeReplicationInfo extracts the RBD volume information from the volumeID, If the +// image is present, mirroring is enabled and the image is in primary state. +func (rs *ReplicationServer) GetVolumeReplicationInfo(ctx context.Context, + req *replication.GetVolumeReplicationInfoRequest, +) (*replication.GetVolumeReplicationInfoResponse, error) { + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "empty volume ID in request") + } + cr, err := util.NewUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + defer cr.DeleteCredentials() + + if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) + } + defer rs.VolumeLocks.Release(volumeID) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + defer rbdVol.Destroy() + if err != nil { + switch { + case errors.Is(err, ErrImageNotFound): + err = status.Errorf(codes.NotFound, "volume %s not found", volumeID) + case errors.Is(err, util.ErrPoolNotFound): + err = status.Errorf(codes.NotFound, "pool %s not found for %s", rbdVol.Pool, volumeID) + default: + err = status.Errorf(codes.Internal, err.Error()) + } + + return nil, err + } + + mirroringInfo, err := rbdVol.getImageMirroringInfo() + if err != nil { + log.ErrorLog(ctx, err.Error()) + + return nil, status.Error(codes.Aborted, err.Error()) + } + + if mirroringInfo.State != librbd.MirrorImageEnabled { + return nil, status.Error(codes.InvalidArgument, "image mirroring is not enabled") + } + + // return error if the image is not in primary state + if !mirroringInfo.Primary { + return nil, status.Error(codes.InvalidArgument, "image is not in primary state") + } + + mirrorStatus, err := rbdVol.getImageMirroringStatus() + if err != nil { + if errors.Is(err, ErrImageNotFound) { + return nil, status.Error(codes.Aborted, err.Error()) + } + log.ErrorLog(ctx, err.Error()) + + return nil, status.Error(codes.Internal, err.Error()) + } + + localStatus, err := mirrorStatus.LocalStatus() + if err != nil { + log.ErrorLog(ctx, err.Error()) + + return nil, fmt.Errorf("failed to get local status: %w", err) + } + + description := localStatus.Description + lastSyncTime, err := getLastSyncTime(description) + if err != nil { + return nil, fmt.Errorf("failed to get last sync time: %w", err) + } + + resp := &replication.GetVolumeReplicationInfoResponse{ + LastSyncTime: lastSyncTime, + } + + return resp, nil +} + +// This function gets the local snapshot time from the description +// of localStatus and converts it into required type. +func getLastSyncTime(description string) (*timestamppb.Timestamp, error) { + // Format of the description will be as followed: + // description = "replaying,{"bytes_per_second":0.0, + // "bytes_per_snapshot":149504.0,"local_snapshot_timestamp":1662655501 + // ,"remote_snapshot_timestamp":1662655501}" + // In case there is no local snapshot timestamp we can pass the default value + if description == "" { + return nil, nil + } + splittedString := strings.SplitN(description, ",", 2) + type localStatus struct { + LocalSnapshotTime int64 `json:"local_snapshot_timestamp"` + } + + var localSnapTime localStatus + err := json.Unmarshal([]byte(splittedString[1]), &localSnapTime) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal description: %w", err) + } + + lastUpdateTime := time.Unix(localSnapTime.LocalSnapshotTime, 0) + lastSyncTime := timestamppb.New(lastUpdateTime) + + return lastSyncTime, nil +} + func resyncVolume(localStatus librbd.SiteMirrorImageStatus, rbdVol *rbdVolume, force bool) error { if resyncRequired(localStatus) { // If the force option is not set return the error message to retry diff --git a/internal/rbd/replicationcontrollerserver_test.go b/internal/rbd/replicationcontrollerserver_test.go index a4b042bb8..d1e3a0041 100644 --- a/internal/rbd/replicationcontrollerserver_test.go +++ b/internal/rbd/replicationcontrollerserver_test.go @@ -19,10 +19,13 @@ package rbd import ( "context" "reflect" + "strings" "testing" + "time" librbd "github.com/ceph/go-ceph/rbd" "github.com/ceph/go-ceph/rbd/admin" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestValidateSchedulingInterval(t *testing.T) { @@ -432,3 +435,54 @@ func TestCheckRemoteSiteStatus(t *testing.T) { }) } } + +func TestValidateLastSyncTime(t *testing.T) { + t.Parallel() + tests := []struct { + name string + description string + timestamp *timestamppb.Timestamp + expectedErr string + }{ + { + "valid description", + //nolint:lll // sample output cannot be split into multiple lines. + `replaying,{"bytes_per_second":0.0,"bytes_per_snapshot":149504.0,"local_snapshot_timestamp":1662655501,"remote_snapshot_timestamp":1662655501}`, + timestamppb.New(time.Unix(1662655501, 0)), + "", + }, + { + "empty description", + "", + nil, + "", + }, + { + "description without local_snapshot_timestamp", + `replaying,{"bytes_per_second":0.0,"bytes_per_snapshot":149504.0,"remote_snapshot_timestamp":1662655501}`, + nil, + "", + }, + { + "description with invalid JSON", + `replaying,{"bytes_per_second":0.0,"bytes_per_snapshot":149504.0","remote_snapshot_timestamp":1662655501`, + nil, + "failed to unmarshal description", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ts, err := getLastSyncTime(tt.description) + if err != nil && !strings.Contains(err.Error(), tt.expectedErr) { + // returned error + t.Errorf("getLastSyncTime() returned error, expected: %v, got: %v", + tt.expectedErr, err) + } + if !ts.AsTime().Equal(tt.timestamp.AsTime()) { + t.Errorf("getLastSyncTime() %v, expected %v", ts, tt.timestamp) + } + }) + } +} diff --git a/vendor/github.com/csi-addons/spec/lib/go/replication/replication.pb.go b/vendor/github.com/csi-addons/spec/lib/go/replication/replication.pb.go index ca3266ca2..6d798eae8 100644 --- a/vendor/github.com/csi-addons/spec/lib/go/replication/replication.pb.go +++ b/vendor/github.com/csi-addons/spec/lib/go/replication/replication.pb.go @@ -14,6 +14,7 @@ import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" descriptorpb "google.golang.org/protobuf/types/descriptorpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" reflect "reflect" sync "sync" ) @@ -705,6 +706,135 @@ func (x *ResyncVolumeResponse) GetReady() bool { return false } +// GetVolumeReplicationInfoRequest holds the required information to get +// the Volume replication information. +type GetVolumeReplicationInfoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // The identifier for this volume, generated by the plugin during + // CreateVolume CSI RPC call. + // This field is REQUIRED. + // This field MUST contain enough information to uniquely identify + // this specific volume vs all other volumes supported by this plugin. + // This field SHALL be used by the CO in subsequent calls to refer to + // this volume. + VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"` + // Secrets required by the plugin to complete the request. + Secrets map[string]string `protobuf:"bytes,2,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // The identifier for the replication. + // This field is OPTIONAL. + // This field MUST contain enough information, together with volume_id, + // to uniquely identify this specific replication + // vs all other replications supported by this plugin. + ReplicationId string `protobuf:"bytes,3,opt,name=replication_id,json=replicationId,proto3" json:"replication_id,omitempty"` +} + +func (x *GetVolumeReplicationInfoRequest) Reset() { + *x = GetVolumeReplicationInfoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_replication_replication_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetVolumeReplicationInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetVolumeReplicationInfoRequest) ProtoMessage() {} + +func (x *GetVolumeReplicationInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_replication_replication_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetVolumeReplicationInfoRequest.ProtoReflect.Descriptor instead. +func (*GetVolumeReplicationInfoRequest) Descriptor() ([]byte, []int) { + return file_replication_replication_proto_rawDescGZIP(), []int{10} +} + +func (x *GetVolumeReplicationInfoRequest) GetVolumeId() string { + if x != nil { + return x.VolumeId + } + return "" +} + +func (x *GetVolumeReplicationInfoRequest) GetSecrets() map[string]string { + if x != nil { + return x.Secrets + } + return nil +} + +func (x *GetVolumeReplicationInfoRequest) GetReplicationId() string { + if x != nil { + return x.ReplicationId + } + return "" +} + +// GetVolumeReplicationInfoResponse holds the information to send the +// volume replication information. +type GetVolumeReplicationInfoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Holds the last sync time. + // This field is REQUIRED. + LastSyncTime *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=last_sync_time,json=lastSyncTime,proto3" json:"last_sync_time,omitempty"` +} + +func (x *GetVolumeReplicationInfoResponse) Reset() { + *x = GetVolumeReplicationInfoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_replication_replication_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetVolumeReplicationInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetVolumeReplicationInfoResponse) ProtoMessage() {} + +func (x *GetVolumeReplicationInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_replication_replication_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetVolumeReplicationInfoResponse.ProtoReflect.Descriptor instead. +func (*GetVolumeReplicationInfoResponse) Descriptor() ([]byte, []int) { + return file_replication_replication_proto_rawDescGZIP(), []int{11} +} + +func (x *GetVolumeReplicationInfoResponse) GetLastSyncTime() *timestamppb.Timestamp { + if x != nil { + return x.LastSyncTime + } + return nil +} + var file_replication_replication_proto_extTypes = []protoimpl.ExtensionInfo{ { ExtendedType: (*descriptorpb.FieldOptions)(nil), @@ -738,132 +868,25 @@ var file_replication_replication_proto_rawDesc = []byte{ 0x2f, 0x63, 0x73, 0x69, 0x2f, 0x63, 0x73, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x9a, 0x03, 0x0a, 0x1e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, - 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x5b, 0x0a, 0x0a, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x3b, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, - 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, - 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x57, 0x0a, 0x07, 0x73, 0x65, 0x63, - 0x72, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, 0x0a, - 0x1f, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x9d, 0x03, 0x0a, 0x1f, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, + 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x22, 0x9a, 0x03, 0x0a, 0x1e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, - 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x5c, 0x0a, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x5b, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x3c, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x07, 0x73, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, - 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, - 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, - 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x22, 0x0a, 0x20, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x92, 0x03, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, - 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, - 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0a, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x31, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, - 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x4d, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, - 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, - 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, - 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x17, 0x0a, 0x15, 0x50, 0x72, 0x6f, - 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x8f, 0x03, 0x0a, 0x13, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x42, - 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x70, 0x61, 0x72, - 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, - 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4c, 0x0a, 0x07, 0x73, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, 0x74, - 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, - 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, - 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, - 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, - 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x16, 0x0a, 0x14, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8f, 0x03, 0x0a, - 0x13, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, - 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, - 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, - 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, - 0x72, 0x63, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, - 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, - 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, - 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4c, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, + 0x0b, 0x32, 0x3b, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, + 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x57, 0x0a, 0x07, 0x73, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x38, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, @@ -872,48 +895,188 @@ var file_replication_replication_proto_rawDesc = []byte{ 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x2c, - 0x0a, 0x14, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x87, 0x04, 0x0a, - 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x76, 0x0a, 0x17, 0x45, - 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, - 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x21, + 0x0a, 0x1f, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, 0x18, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, - 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, - 0x2c, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x69, - 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, + 0x65, 0x22, 0x9d, 0x03, 0x0a, 0x1f, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, + 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x5c, + 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x07, + 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x58, - 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, - 0x21, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, - 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x2e, 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x44, 0x65, 0x6d, 0x6f, - 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, - 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, - 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, - 0x55, 0x0a, 0x0c, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, - 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, - 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x21, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x3a, 0x3f, 0x0a, 0x0b, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x5f, - 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xcc, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x42, 0x0f, 0x5a, 0x0d, 0x2e, 0x3b, 0x72, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, + 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, + 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, + 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x22, 0x0a, 0x20, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x92, 0x03, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, + 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, + 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x51, 0x0a, + 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x31, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x12, 0x4d, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x2e, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, + 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, + 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x17, 0x0a, 0x15, 0x50, 0x72, + 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x8f, 0x03, 0x0a, 0x13, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x70, 0x61, + 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, + 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, + 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4c, 0x0a, 0x07, + 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, + 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, + 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, + 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x16, 0x0a, 0x14, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x8f, 0x03, + 0x0a, 0x13, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, + 0x49, 0x64, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, + 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x14, + 0x0a, 0x05, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, + 0x6f, 0x72, 0x63, 0x65, 0x12, 0x50, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x30, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50, 0x61, 0x72, 0x61, 0x6d, + 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x61, + 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x4c, 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x07, 0x73, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x73, 0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0x2c, 0x0a, 0x14, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x80, 0x02, + 0x0a, 0x1f, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x58, + 0x0a, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x39, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x47, 0x65, + 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53, 0x65, + 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, + 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x12, 0x2a, 0x0a, 0x0e, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x42, 0x03, 0xe0, 0x44, 0x01, 0x52, 0x0d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x1a, 0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x64, 0x0a, 0x20, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x40, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x73, 0x79, 0x6e, + 0x63, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0c, 0x6c, 0x61, 0x73, 0x74, 0x53, 0x79, + 0x6e, 0x63, 0x54, 0x69, 0x6d, 0x65, 0x32, 0x82, 0x05, 0x0a, 0x0a, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x76, 0x0a, 0x17, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x2b, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, + 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, + 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x61, 0x62, + 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x79, 0x0a, + 0x18, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x2e, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x56, 0x6f, 0x6c, + 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x58, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6d, + 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x21, 0x2e, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x6d, 0x6f, 0x74, 0x65, 0x56, + 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x6d, 0x6f, + 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x44, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x55, 0x0a, 0x0c, 0x52, 0x65, 0x73, + 0x79, 0x6e, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x72, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x72, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, + 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x79, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x2c, 0x2e, 0x72, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x72, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x47, 0x65, 0x74, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x3a, 0x3f, 0x0a, 0x0b, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0xcc, 0x08, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x0a, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x42, 0x0f, 0x5a, 0x0d, + 0x2e, 0x3b, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -928,7 +1091,7 @@ func file_replication_replication_proto_rawDescGZIP() []byte { return file_replication_replication_proto_rawDescData } -var file_replication_replication_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_replication_replication_proto_msgTypes = make([]protoimpl.MessageInfo, 23) var file_replication_replication_proto_goTypes = []interface{}{ (*EnableVolumeReplicationRequest)(nil), // 0: replication.EnableVolumeReplicationRequest (*EnableVolumeReplicationResponse)(nil), // 1: replication.EnableVolumeReplicationResponse @@ -940,45 +1103,53 @@ var file_replication_replication_proto_goTypes = []interface{}{ (*DemoteVolumeResponse)(nil), // 7: replication.DemoteVolumeResponse (*ResyncVolumeRequest)(nil), // 8: replication.ResyncVolumeRequest (*ResyncVolumeResponse)(nil), // 9: replication.ResyncVolumeResponse - nil, // 10: replication.EnableVolumeReplicationRequest.ParametersEntry - nil, // 11: replication.EnableVolumeReplicationRequest.SecretsEntry - nil, // 12: replication.DisableVolumeReplicationRequest.ParametersEntry - nil, // 13: replication.DisableVolumeReplicationRequest.SecretsEntry - nil, // 14: replication.PromoteVolumeRequest.ParametersEntry - nil, // 15: replication.PromoteVolumeRequest.SecretsEntry - nil, // 16: replication.DemoteVolumeRequest.ParametersEntry - nil, // 17: replication.DemoteVolumeRequest.SecretsEntry - nil, // 18: replication.ResyncVolumeRequest.ParametersEntry - nil, // 19: replication.ResyncVolumeRequest.SecretsEntry - (*descriptorpb.FieldOptions)(nil), // 20: google.protobuf.FieldOptions + (*GetVolumeReplicationInfoRequest)(nil), // 10: replication.GetVolumeReplicationInfoRequest + (*GetVolumeReplicationInfoResponse)(nil), // 11: replication.GetVolumeReplicationInfoResponse + nil, // 12: replication.EnableVolumeReplicationRequest.ParametersEntry + nil, // 13: replication.EnableVolumeReplicationRequest.SecretsEntry + nil, // 14: replication.DisableVolumeReplicationRequest.ParametersEntry + nil, // 15: replication.DisableVolumeReplicationRequest.SecretsEntry + nil, // 16: replication.PromoteVolumeRequest.ParametersEntry + nil, // 17: replication.PromoteVolumeRequest.SecretsEntry + nil, // 18: replication.DemoteVolumeRequest.ParametersEntry + nil, // 19: replication.DemoteVolumeRequest.SecretsEntry + nil, // 20: replication.ResyncVolumeRequest.ParametersEntry + nil, // 21: replication.ResyncVolumeRequest.SecretsEntry + nil, // 22: replication.GetVolumeReplicationInfoRequest.SecretsEntry + (*timestamppb.Timestamp)(nil), // 23: google.protobuf.Timestamp + (*descriptorpb.FieldOptions)(nil), // 24: google.protobuf.FieldOptions } var file_replication_replication_proto_depIdxs = []int32{ - 10, // 0: replication.EnableVolumeReplicationRequest.parameters:type_name -> replication.EnableVolumeReplicationRequest.ParametersEntry - 11, // 1: replication.EnableVolumeReplicationRequest.secrets:type_name -> replication.EnableVolumeReplicationRequest.SecretsEntry - 12, // 2: replication.DisableVolumeReplicationRequest.parameters:type_name -> replication.DisableVolumeReplicationRequest.ParametersEntry - 13, // 3: replication.DisableVolumeReplicationRequest.secrets:type_name -> replication.DisableVolumeReplicationRequest.SecretsEntry - 14, // 4: replication.PromoteVolumeRequest.parameters:type_name -> replication.PromoteVolumeRequest.ParametersEntry - 15, // 5: replication.PromoteVolumeRequest.secrets:type_name -> replication.PromoteVolumeRequest.SecretsEntry - 16, // 6: replication.DemoteVolumeRequest.parameters:type_name -> replication.DemoteVolumeRequest.ParametersEntry - 17, // 7: replication.DemoteVolumeRequest.secrets:type_name -> replication.DemoteVolumeRequest.SecretsEntry - 18, // 8: replication.ResyncVolumeRequest.parameters:type_name -> replication.ResyncVolumeRequest.ParametersEntry - 19, // 9: replication.ResyncVolumeRequest.secrets:type_name -> replication.ResyncVolumeRequest.SecretsEntry - 20, // 10: replication.alpha_field:extendee -> google.protobuf.FieldOptions - 0, // 11: replication.Controller.EnableVolumeReplication:input_type -> replication.EnableVolumeReplicationRequest - 2, // 12: replication.Controller.DisableVolumeReplication:input_type -> replication.DisableVolumeReplicationRequest - 4, // 13: replication.Controller.PromoteVolume:input_type -> replication.PromoteVolumeRequest - 6, // 14: replication.Controller.DemoteVolume:input_type -> replication.DemoteVolumeRequest - 8, // 15: replication.Controller.ResyncVolume:input_type -> replication.ResyncVolumeRequest - 1, // 16: replication.Controller.EnableVolumeReplication:output_type -> replication.EnableVolumeReplicationResponse - 3, // 17: replication.Controller.DisableVolumeReplication:output_type -> replication.DisableVolumeReplicationResponse - 5, // 18: replication.Controller.PromoteVolume:output_type -> replication.PromoteVolumeResponse - 7, // 19: replication.Controller.DemoteVolume:output_type -> replication.DemoteVolumeResponse - 9, // 20: replication.Controller.ResyncVolume:output_type -> replication.ResyncVolumeResponse - 16, // [16:21] is the sub-list for method output_type - 11, // [11:16] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 10, // [10:11] is the sub-list for extension extendee - 0, // [0:10] is the sub-list for field type_name + 12, // 0: replication.EnableVolumeReplicationRequest.parameters:type_name -> replication.EnableVolumeReplicationRequest.ParametersEntry + 13, // 1: replication.EnableVolumeReplicationRequest.secrets:type_name -> replication.EnableVolumeReplicationRequest.SecretsEntry + 14, // 2: replication.DisableVolumeReplicationRequest.parameters:type_name -> replication.DisableVolumeReplicationRequest.ParametersEntry + 15, // 3: replication.DisableVolumeReplicationRequest.secrets:type_name -> replication.DisableVolumeReplicationRequest.SecretsEntry + 16, // 4: replication.PromoteVolumeRequest.parameters:type_name -> replication.PromoteVolumeRequest.ParametersEntry + 17, // 5: replication.PromoteVolumeRequest.secrets:type_name -> replication.PromoteVolumeRequest.SecretsEntry + 18, // 6: replication.DemoteVolumeRequest.parameters:type_name -> replication.DemoteVolumeRequest.ParametersEntry + 19, // 7: replication.DemoteVolumeRequest.secrets:type_name -> replication.DemoteVolumeRequest.SecretsEntry + 20, // 8: replication.ResyncVolumeRequest.parameters:type_name -> replication.ResyncVolumeRequest.ParametersEntry + 21, // 9: replication.ResyncVolumeRequest.secrets:type_name -> replication.ResyncVolumeRequest.SecretsEntry + 22, // 10: replication.GetVolumeReplicationInfoRequest.secrets:type_name -> replication.GetVolumeReplicationInfoRequest.SecretsEntry + 23, // 11: replication.GetVolumeReplicationInfoResponse.last_sync_time:type_name -> google.protobuf.Timestamp + 24, // 12: replication.alpha_field:extendee -> google.protobuf.FieldOptions + 0, // 13: replication.Controller.EnableVolumeReplication:input_type -> replication.EnableVolumeReplicationRequest + 2, // 14: replication.Controller.DisableVolumeReplication:input_type -> replication.DisableVolumeReplicationRequest + 4, // 15: replication.Controller.PromoteVolume:input_type -> replication.PromoteVolumeRequest + 6, // 16: replication.Controller.DemoteVolume:input_type -> replication.DemoteVolumeRequest + 8, // 17: replication.Controller.ResyncVolume:input_type -> replication.ResyncVolumeRequest + 10, // 18: replication.Controller.GetVolumeReplicationInfo:input_type -> replication.GetVolumeReplicationInfoRequest + 1, // 19: replication.Controller.EnableVolumeReplication:output_type -> replication.EnableVolumeReplicationResponse + 3, // 20: replication.Controller.DisableVolumeReplication:output_type -> replication.DisableVolumeReplicationResponse + 5, // 21: replication.Controller.PromoteVolume:output_type -> replication.PromoteVolumeResponse + 7, // 22: replication.Controller.DemoteVolume:output_type -> replication.DemoteVolumeResponse + 9, // 23: replication.Controller.ResyncVolume:output_type -> replication.ResyncVolumeResponse + 11, // 24: replication.Controller.GetVolumeReplicationInfo:output_type -> replication.GetVolumeReplicationInfoResponse + 19, // [19:25] is the sub-list for method output_type + 13, // [13:19] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 12, // [12:13] is the sub-list for extension extendee + 0, // [0:12] is the sub-list for field type_name } func init() { file_replication_replication_proto_init() } @@ -1107,6 +1278,30 @@ func file_replication_replication_proto_init() { return nil } } + file_replication_replication_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetVolumeReplicationInfoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_replication_replication_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetVolumeReplicationInfoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -1114,7 +1309,7 @@ func file_replication_replication_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_replication_replication_proto_rawDesc, NumEnums: 0, - NumMessages: 20, + NumMessages: 23, NumExtensions: 1, NumServices: 1, }, diff --git a/vendor/github.com/csi-addons/spec/lib/go/replication/replication_grpc.pb.go b/vendor/github.com/csi-addons/spec/lib/go/replication/replication_grpc.pb.go index 87c8be8a3..aa9c826bd 100644 --- a/vendor/github.com/csi-addons/spec/lib/go/replication/replication_grpc.pb.go +++ b/vendor/github.com/csi-addons/spec/lib/go/replication/replication_grpc.pb.go @@ -28,6 +28,9 @@ type ControllerClient interface { DemoteVolume(ctx context.Context, in *DemoteVolumeRequest, opts ...grpc.CallOption) (*DemoteVolumeResponse, error) // ResyncVolume RPC call to resync the volume. ResyncVolume(ctx context.Context, in *ResyncVolumeRequest, opts ...grpc.CallOption) (*ResyncVolumeResponse, error) + // GetVolumeReplicationInfo RPC call to get the volume replication + // information. + GetVolumeReplicationInfo(ctx context.Context, in *GetVolumeReplicationInfoRequest, opts ...grpc.CallOption) (*GetVolumeReplicationInfoResponse, error) } type controllerClient struct { @@ -83,6 +86,15 @@ func (c *controllerClient) ResyncVolume(ctx context.Context, in *ResyncVolumeReq return out, nil } +func (c *controllerClient) GetVolumeReplicationInfo(ctx context.Context, in *GetVolumeReplicationInfoRequest, opts ...grpc.CallOption) (*GetVolumeReplicationInfoResponse, error) { + out := new(GetVolumeReplicationInfoResponse) + err := c.cc.Invoke(ctx, "/replication.Controller/GetVolumeReplicationInfo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // ControllerServer is the server API for Controller service. // All implementations must embed UnimplementedControllerServer // for forward compatibility @@ -97,6 +109,9 @@ type ControllerServer interface { DemoteVolume(context.Context, *DemoteVolumeRequest) (*DemoteVolumeResponse, error) // ResyncVolume RPC call to resync the volume. ResyncVolume(context.Context, *ResyncVolumeRequest) (*ResyncVolumeResponse, error) + // GetVolumeReplicationInfo RPC call to get the volume replication + // information. + GetVolumeReplicationInfo(context.Context, *GetVolumeReplicationInfoRequest) (*GetVolumeReplicationInfoResponse, error) mustEmbedUnimplementedControllerServer() } @@ -119,6 +134,9 @@ func (UnimplementedControllerServer) DemoteVolume(context.Context, *DemoteVolume func (UnimplementedControllerServer) ResyncVolume(context.Context, *ResyncVolumeRequest) (*ResyncVolumeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ResyncVolume not implemented") } +func (UnimplementedControllerServer) GetVolumeReplicationInfo(context.Context, *GetVolumeReplicationInfoRequest) (*GetVolumeReplicationInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVolumeReplicationInfo not implemented") +} func (UnimplementedControllerServer) mustEmbedUnimplementedControllerServer() {} // UnsafeControllerServer may be embedded to opt out of forward compatibility for this service. @@ -222,6 +240,24 @@ func _Controller_ResyncVolume_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _Controller_GetVolumeReplicationInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVolumeReplicationInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControllerServer).GetVolumeReplicationInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/replication.Controller/GetVolumeReplicationInfo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControllerServer).GetVolumeReplicationInfo(ctx, req.(*GetVolumeReplicationInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Controller_ServiceDesc is the grpc.ServiceDesc for Controller service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -249,6 +285,10 @@ var Controller_ServiceDesc = grpc.ServiceDesc{ MethodName: "ResyncVolume", Handler: _Controller_ResyncVolume_Handler, }, + { + MethodName: "GetVolumeReplicationInfo", + Handler: _Controller_GetVolumeReplicationInfo_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "replication/replication.proto", diff --git a/vendor/modules.txt b/vendor/modules.txt index 5784e1436..d709fa00f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -150,7 +150,7 @@ github.com/container-storage-interface/spec/lib/go/csi # github.com/csi-addons/replication-lib-utils v0.2.0 ## explicit; go 1.15 github.com/csi-addons/replication-lib-utils/protosanitizer -# github.com/csi-addons/spec v0.1.2-0.20220829042231-b27a0d84b50b +# github.com/csi-addons/spec v0.1.2-0.20220906123848-52ce69f90900 ## explicit github.com/csi-addons/spec/lib/go/fence github.com/csi-addons/spec/lib/go/identity