Merge pull request #190 from ceph/devel

Sync the upstream changes from ceph/ceph-csi:devel into the devel branch.

.github/dependabot.yml

@@ -13,6 +13,7 @@ updates:
       golang-dependencies:
         patterns:
           - "github.com/golang*"
+          - "golang.org/x/*"
       k8s-dependencies:
         patterns:
           - "k8s.io*"

.github/workflows/build-multi-stage.yaml

@@ -13,7 +13,7 @@ jobs:
     name: multi-arch-build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: multi-arch-build
         # yamllint disable-line rule:line-length
         if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ci/skip/multi-arch-build') }}

.github/workflows/codespell.yaml

@@ -15,6 +15,6 @@ jobs:
     name: codespell
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: codespell
         run: make containerized-test TARGET=codespell

.github/workflows/commitlint.yaml

@@ -14,7 +14,7 @@ jobs:
     if: ${{ github.event.pull_request.user.login != 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: commitlint

.github/workflows/dependency-review.yaml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Dependency Review'
         uses: actions/dependency-review-action@v3
         with:

.github/workflows/go-test.yaml

@@ -13,13 +13,13 @@ jobs:
     name: go-test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: go-test
         run: make containerized-test TARGET=go-test
   go-test-api:
     name: go-test-api
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: go-test-api
         run: make containerized-test TARGET=go-test-api

.github/workflows/golangci-lint.yaml

@@ -13,6 +13,6 @@ jobs:
     name: golangci-lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: golangci-lint
         run: make containerized-test TARGET=go-lint

.github/workflows/lint-extras.yaml

@@ -13,6 +13,6 @@ jobs:
     name: lint-extras
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: lint-extras
         run: make containerized-test TARGET=lint-extras

.github/workflows/mod-check.yaml

@@ -13,6 +13,6 @@ jobs:
     name: mod-check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: mod-check
         run: make containerized-test TARGET=mod-check

.github/workflows/publish-artifacts.yaml

@@ -18,10 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     if: github.repository == 'ceph/ceph-csi'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Login to Quay
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_IO_USERNAME }}

.github/workflows/test-retest-action.yaml

@@ -15,7 +15,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Docker build
         # Run cd to avoid loading complete cephcsi directory in docker context

actions/retest/go.mod

@@ -4,13 +4,13 @@ go 1.18
 
 require (
 	github.com/google/go-github v17.0.0+incompatible
-	golang.org/x/oauth2 v0.11.0
+	golang.org/x/oauth2 v0.13.0
 )
 
 require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
+	golang.org/x/net v0.16.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 )

actions/retest/go.sum

@@ -11,10 +11,10 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
-golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
+golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
+golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
+golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

actions/retest/vendor/golang.org/x/oauth2/deviceauth.go

@@ -0,0 +1,198 @@
+package oauth2
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"golang.org/x/oauth2/internal"
+)
+
+// https://datatracker.ietf.org/doc/html/rfc8628#section-3.5
+const (
+	errAuthorizationPending = "authorization_pending"
+	errSlowDown             = "slow_down"
+	errAccessDenied         = "access_denied"
+	errExpiredToken         = "expired_token"
+)
+
+// DeviceAuthResponse describes a successful RFC 8628 Device Authorization Response
+// https://datatracker.ietf.org/doc/html/rfc8628#section-3.2
+type DeviceAuthResponse struct {
+	// DeviceCode
+	DeviceCode string `json:"device_code"`
+	// UserCode is the code the user should enter at the verification uri
+	UserCode string `json:"user_code"`
+	// VerificationURI is where user should enter the user code
+	VerificationURI string `json:"verification_uri"`
+	// VerificationURIComplete (if populated) includes the user code in the verification URI. This is typically shown to the user in non-textual form, such as a QR code.
+	VerificationURIComplete string `json:"verification_uri_complete,omitempty"`
+	// Expiry is when the device code and user code expire
+	Expiry time.Time `json:"expires_in,omitempty"`
+	// Interval is the duration in seconds that Poll should wait between requests
+	Interval int64 `json:"interval,omitempty"`
+}
+
+func (d DeviceAuthResponse) MarshalJSON() ([]byte, error) {
+	type Alias DeviceAuthResponse
+	var expiresIn int64
+	if !d.Expiry.IsZero() {
+		expiresIn = int64(time.Until(d.Expiry).Seconds())
+	}
+	return json.Marshal(&struct {
+		ExpiresIn int64 `json:"expires_in,omitempty"`
+		*Alias
+	}{
+		ExpiresIn: expiresIn,
+		Alias:     (*Alias)(&d),
+	})
+
+}
+
+func (c *DeviceAuthResponse) UnmarshalJSON(data []byte) error {
+	type Alias DeviceAuthResponse
+	aux := &struct {
+		ExpiresIn int64 `json:"expires_in"`
+		// workaround misspelling of verification_uri
+		VerificationURL string `json:"verification_url"`
+		*Alias
+	}{
+		Alias: (*Alias)(c),
+	}
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+	if aux.ExpiresIn != 0 {
+		c.Expiry = time.Now().UTC().Add(time.Second * time.Duration(aux.ExpiresIn))
+	}
+	if c.VerificationURI == "" {
+		c.VerificationURI = aux.VerificationURL
+	}
+	return nil
+}
+
+// DeviceAuth returns a device auth struct which contains a device code
+// and authorization information provided for users to enter on another device.
+func (c *Config) DeviceAuth(ctx context.Context, opts ...AuthCodeOption) (*DeviceAuthResponse, error) {
+	// https://datatracker.ietf.org/doc/html/rfc8628#section-3.1
+	v := url.Values{
+		"client_id": {c.ClientID},
+	}
+	if len(c.Scopes) > 0 {
+		v.Set("scope", strings.Join(c.Scopes, " "))
+	}
+	for _, opt := range opts {
+		opt.setValue(v)
+	}
+	return retrieveDeviceAuth(ctx, c, v)
+}
+
+func retrieveDeviceAuth(ctx context.Context, c *Config, v url.Values) (*DeviceAuthResponse, error) {
+	if c.Endpoint.DeviceAuthURL == "" {
+		return nil, errors.New("endpoint missing DeviceAuthURL")
+	}
+
+	req, err := http.NewRequest("POST", c.Endpoint.DeviceAuthURL, strings.NewReader(v.Encode()))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+	req.Header.Set("Accept", "application/json")
+
+	t := time.Now()
+	r, err := internal.ContextClient(ctx).Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20))
+	if err != nil {
+		return nil, fmt.Errorf("oauth2: cannot auth device: %v", err)
+	}
+	if code := r.StatusCode; code < 200 || code > 299 {
+		return nil, &RetrieveError{
+			Response: r,
+			Body:     body,
+		}
+	}
+
+	da := &DeviceAuthResponse{}
+	err = json.Unmarshal(body, &da)
+	if err != nil {
+		return nil, fmt.Errorf("unmarshal %s", err)
+	}
+
+	if !da.Expiry.IsZero() {
+		// Make a small adjustment to account for time taken by the request
+		da.Expiry = da.Expiry.Add(-time.Since(t))
+	}
+
+	return da, nil
+}
+
+// DeviceAccessToken polls the server to exchange a device code for a token.
+func (c *Config) DeviceAccessToken(ctx context.Context, da *DeviceAuthResponse, opts ...AuthCodeOption) (*Token, error) {
+	if !da.Expiry.IsZero() {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithDeadline(ctx, da.Expiry)
+		defer cancel()
+	}
+
+	// https://datatracker.ietf.org/doc/html/rfc8628#section-3.4
+	v := url.Values{
+		"client_id":   {c.ClientID},
+		"grant_type":  {"urn:ietf:params:oauth:grant-type:device_code"},
+		"device_code": {da.DeviceCode},
+	}
+	if len(c.Scopes) > 0 {
+		v.Set("scope", strings.Join(c.Scopes, " "))
+	}
+	for _, opt := range opts {
+		opt.setValue(v)
+	}
+
+	// "If no value is provided, clients MUST use 5 as the default."
+	// https://datatracker.ietf.org/doc/html/rfc8628#section-3.2
+	interval := da.Interval
+	if interval == 0 {
+		interval = 5
+	}
+
+	ticker := time.NewTicker(time.Duration(interval) * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-ticker.C:
+			tok, err := retrieveToken(ctx, c, v)
+			if err == nil {
+				return tok, nil
+			}
+
+			e, ok := err.(*RetrieveError)
+			if !ok {
+				return nil, err
+			}
+			switch e.ErrorCode {
+			case errSlowDown:
+				// https://datatracker.ietf.org/doc/html/rfc8628#section-3.5
+				// "the interval MUST be increased by 5 seconds for this and all subsequent requests"
+				interval += 5
+				ticker.Reset(time.Duration(interval) * time.Second)
+			case errAuthorizationPending:
+				// Do nothing.
+			case errAccessDenied, errExpiredToken:
+				fallthrough
+			default:
+				return tok, err
+			}
+		}
+	}
+}

actions/retest/vendor/golang.org/x/oauth2/internal/token.go

@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -115,41 +116,60 @@ const (
 	AuthStyleInHeader AuthStyle = 2
 )
 
-// authStyleCache is the set of tokenURLs we've successfully used via
+// LazyAuthStyleCache is a backwards compatibility compromise to let Configs
+// have a lazily-initialized AuthStyleCache.
+//
+// The two users of this, oauth2.Config and oauth2/clientcredentials.Config,
+// both would ideally just embed an unexported AuthStyleCache but because both
+// were historically allowed to be copied by value we can't retroactively add an
+// uncopyable Mutex to them.
+//
+// We could use an atomic.Pointer, but that was added recently enough (in Go
+// 1.18) that we'd break Go 1.17 users where the tests as of 2023-08-03
+// still pass. By using an atomic.Value, it supports both Go 1.17 and
+// copying by value, even if that's not ideal.
+type LazyAuthStyleCache struct {
+	v atomic.Value // of *AuthStyleCache
+}
+
+func (lc *LazyAuthStyleCache) Get() *AuthStyleCache {
+	if c, ok := lc.v.Load().(*AuthStyleCache); ok {
+		return c
+	}
+	c := new(AuthStyleCache)
+	if !lc.v.CompareAndSwap(nil, c) {
+		c = lc.v.Load().(*AuthStyleCache)
+	}
+	return c
+}
+
+// AuthStyleCache is the set of tokenURLs we've successfully used via
 // RetrieveToken and which style auth we ended up using.
 // It's called a cache, but it doesn't (yet?) shrink. It's expected that
 // the set of OAuth2 servers a program contacts over time is fixed and
 // small.
-var authStyleCache struct {
-	sync.Mutex
-	m map[string]AuthStyle // keyed by tokenURL
-}
-
-// ResetAuthCache resets the global authentication style cache used
-// for AuthStyleUnknown token requests.
-func ResetAuthCache() {
-	authStyleCache.Lock()
-	defer authStyleCache.Unlock()
-	authStyleCache.m = nil
+type AuthStyleCache struct {
+	mu sync.Mutex
+	m  map[string]AuthStyle // keyed by tokenURL
 }
 
 // lookupAuthStyle reports which auth style we last used with tokenURL
 // when calling RetrieveToken and whether we have ever done so.
-func lookupAuthStyle(tokenURL string) (style AuthStyle, ok bool) {
-	authStyleCache.Lock()
-	defer authStyleCache.Unlock()
-	style, ok = authStyleCache.m[tokenURL]
+func (c *AuthStyleCache) lookupAuthStyle(tokenURL string) (style AuthStyle, ok bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	style, ok = c.m[tokenURL]
 	return
 }
 
 // setAuthStyle adds an entry to authStyleCache, documented above.
-func setAuthStyle(tokenURL string, v AuthStyle) {
-	authStyleCache.Lock()
-	defer authStyleCache.Unlock()
-	if authStyleCache.m == nil {
-		authStyleCache.m = make(map[string]AuthStyle)
+func (c *AuthStyleCache) setAuthStyle(tokenURL string, v AuthStyle) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if c.m == nil {
+		c.m = make(map[string]AuthStyle)
 	}
-	authStyleCache.m[tokenURL] = v
+	c.m[tokenURL] = v
 }
 
 // newTokenRequest returns a new *http.Request to retrieve a new token
@@ -189,10 +209,10 @@ func cloneURLValues(v url.Values) url.Values {
 	return v2
 }
 
-func RetrieveToken(ctx context.Context, clientID, clientSecret, tokenURL string, v url.Values, authStyle AuthStyle) (*Token, error) {
+func RetrieveToken(ctx context.Context, clientID, clientSecret, tokenURL string, v url.Values, authStyle AuthStyle, styleCache *AuthStyleCache) (*Token, error) {
 	needsAuthStyleProbe := authStyle == 0
 	if needsAuthStyleProbe {
-		if style, ok := lookupAuthStyle(tokenURL); ok {
+		if style, ok := styleCache.lookupAuthStyle(tokenURL); ok {
 			authStyle = style
 			needsAuthStyleProbe = false
 		} else {
@@ -222,7 +242,7 @@ func RetrieveToken(ctx context.Context, clientID, clientSecret, tokenURL string,
 		token, err = doTokenRoundTrip(ctx, req)
 	}
 	if needsAuthStyleProbe && err == nil {
-		setAuthStyle(tokenURL, authStyle)
+		styleCache.setAuthStyle(tokenURL, authStyle)
 	}
 	// Don't overwrite `RefreshToken` with an empty value
 	// if this was a token refreshing request.

actions/retest/vendor/golang.org/x/oauth2/oauth2.go

@@ -58,6 +58,10 @@ type Config struct {
 
 	// Scope specifies optional requested permissions.
 	Scopes []string
+
+	// authStyleCache caches which auth style to use when Endpoint.AuthStyle is
+	// the zero value (AuthStyleAutoDetect).
+	authStyleCache internal.LazyAuthStyleCache
 }
 
 // A TokenSource is anything that can return a token.
@@ -71,8 +75,9 @@ type TokenSource interface {
 // Endpoint represents an OAuth 2.0 provider's authorization and token
 // endpoint URLs.
 type Endpoint struct {
-	AuthURL  string
-	TokenURL string
+	AuthURL       string
+	DeviceAuthURL string
+	TokenURL      string
 
 	// AuthStyle optionally specifies how the endpoint wants the
 	// client ID & client secret sent. The zero value means to
@@ -139,15 +144,19 @@ func SetAuthURLParam(key, value string) AuthCodeOption {
 // AuthCodeURL returns a URL to OAuth 2.0 provider's consent page
 // that asks for permissions for the required scopes explicitly.
 //
-// State is a token to protect the user from CSRF attacks. You must
-// always provide a non-empty string and validate that it matches the
-// state query parameter on your redirect callback.
-// See http://tools.ietf.org/html/rfc6749#section-10.12 for more info.
+// State is an opaque value used by the client to maintain state between the
+// request and callback. The authorization server includes this value when
+// redirecting the user agent back to the client.
 //
 // Opts may include AccessTypeOnline or AccessTypeOffline, as well
 // as ApprovalForce.
-// It can also be used to pass the PKCE challenge.
-// See https://www.oauth.com/oauth2-servers/pkce/ for more info.
+//
+// To protect against CSRF attacks, opts should include a PKCE challenge
+// (S256ChallengeOption). Not all servers support PKCE. An alternative is to
+// generate a random state parameter and verify it after exchange.
+// See https://datatracker.ietf.org/doc/html/rfc6749#section-10.12 (predating
+// PKCE), https://www.oauth.com/oauth2-servers/pkce/ and
+// https://www.ietf.org/archive/id/draft-ietf-oauth-v2-1-09.html#name-cross-site-request-forgery (describing both approaches)
 func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string {
 	var buf bytes.Buffer
 	buf.WriteString(c.Endpoint.AuthURL)
@@ -162,7 +171,6 @@ func (c *Config) AuthCodeURL(state string, opts ...AuthCodeOption) string {
 		v.Set("scope", strings.Join(c.Scopes, " "))
 	}
 	if state != "" {
-		// TODO(light): Docs say never to omit state; don't allow empty.
 		v.Set("state", state)
 	}
 	for _, opt := range opts {
@@ -207,10 +215,11 @@ func (c *Config) PasswordCredentialsToken(ctx context.Context, username, passwor
 // The provided context optionally controls which HTTP client is used. See the HTTPClient variable.
 //
 // The code will be in the *http.Request.FormValue("code"). Before
-// calling Exchange, be sure to validate FormValue("state").
+// calling Exchange, be sure to validate FormValue("state") if you are
+// using it to protect against CSRF attacks.
 //
-// Opts may include the PKCE verifier code if previously used in AuthCodeURL.
-// See https://www.oauth.com/oauth2-servers/pkce/ for more info.
+// If using PKCE to protect against CSRF attacks, opts should include a
+// VerifierOption.
 func (c *Config) Exchange(ctx context.Context, code string, opts ...AuthCodeOption) (*Token, error) {
 	v := url.Values{
 		"grant_type": {"authorization_code"},

actions/retest/vendor/golang.org/x/oauth2/pkce.go

@@ -0,0 +1,68 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+package oauth2
+
+import (
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"net/url"
+)
+
+const (
+	codeChallengeKey       = "code_challenge"
+	codeChallengeMethodKey = "code_challenge_method"
+	codeVerifierKey        = "code_verifier"
+)
+
+// GenerateVerifier generates a PKCE code verifier with 32 octets of randomness.
+// This follows recommendations in RFC 7636.
+//
+// A fresh verifier should be generated for each authorization.
+// S256ChallengeOption(verifier) should then be passed to Config.AuthCodeURL
+// (or Config.DeviceAccess) and VerifierOption(verifier) to Config.Exchange
+// (or Config.DeviceAccessToken).
+func GenerateVerifier() string {
+	// "RECOMMENDED that the output of a suitable random number generator be
+	// used to create a 32-octet sequence.  The octet sequence is then
+	// base64url-encoded to produce a 43-octet URL-safe string to use as the
+	// code verifier."
+	// https://datatracker.ietf.org/doc/html/rfc7636#section-4.1
+	data := make([]byte, 32)
+	if _, err := rand.Read(data); err != nil {
+		panic(err)
+	}
+	return base64.RawURLEncoding.EncodeToString(data)
+}
+
+// VerifierOption returns a PKCE code verifier AuthCodeOption. It should be
+// passed to Config.Exchange or Config.DeviceAccessToken only.
+func VerifierOption(verifier string) AuthCodeOption {
+	return setParam{k: codeVerifierKey, v: verifier}
+}
+
+// S256ChallengeFromVerifier returns a PKCE code challenge derived from verifier with method S256.
+//
+// Prefer to use S256ChallengeOption where possible.
+func S256ChallengeFromVerifier(verifier string) string {
+	sha := sha256.Sum256([]byte(verifier))
+	return base64.RawURLEncoding.EncodeToString(sha[:])
+}
+
+// S256ChallengeOption derives a PKCE code challenge derived from verifier with
+// method S256. It should be passed to Config.AuthCodeURL or Config.DeviceAccess
+// only.
+func S256ChallengeOption(verifier string) AuthCodeOption {
+	return challengeOption{
+		challenge_method: "S256",
+		challenge:        S256ChallengeFromVerifier(verifier),
+	}
+}
+
+type challengeOption struct{ challenge_method, challenge string }
+
+func (p challengeOption) setValue(m url.Values) {
+	m.Set(codeChallengeMethodKey, p.challenge_method)
+	m.Set(codeChallengeKey, p.challenge)
+}

actions/retest/vendor/golang.org/x/oauth2/token.go

@@ -164,7 +164,7 @@ func tokenFromInternal(t *internal.Token) *Token {
 // This token is then mapped from *internal.Token into an *oauth2.Token which is returned along
 // with an error..
 func retrieveToken(ctx context.Context, c *Config, v url.Values) (*Token, error) {
-	tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v, internal.AuthStyle(c.Endpoint.AuthStyle))
+	tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v, internal.AuthStyle(c.Endpoint.AuthStyle), c.authStyleCache.Get())
 	if err != nil {
 		if rErr, ok := err.(*internal.RetrieveError); ok {
 			return nil, (*RetrieveError)(rErr)

actions/retest/vendor/modules.txt

@@ -7,10 +7,10 @@ github.com/google/go-github/github
 # github.com/google/go-querystring v1.1.0
 ## explicit; go 1.10
 github.com/google/go-querystring/query
-# golang.org/x/net v0.14.0
+# golang.org/x/net v0.16.0
 ## explicit; go 1.17
 golang.org/x/net/context
-# golang.org/x/oauth2 v0.11.0
+# golang.org/x/oauth2 v0.13.0
 ## explicit; go 1.18
 golang.org/x/oauth2
 golang.org/x/oauth2/internal

api/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/openshift/api v0.0.0-20230320192226-1fc631efd341
 	github.com/stretchr/testify v1.8.4
-	k8s.io/api v0.28.1
+	k8s.io/api v0.28.2
 )
 
 require (
@@ -23,7 +23,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apimachinery v0.28.1 // indirect
+	k8s.io/apimachinery v0.28.2 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

api/go.sum

@@ -73,10 +73,10 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108=
-k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg=
-k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY=
-k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw=
+k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=

build.env

@@ -54,11 +54,11 @@ ROOK_VERSION=v1.12.1
 ROOK_CEPH_CLUSTER_IMAGE=quay.io/ceph/ceph:v18
 
 # CSI sidecar version
-CSI_ATTACHER_VERSION=v4.3.0
-CSI_SNAPSHOTTER_VERSION=v6.2.2
-CSI_RESIZER_VERSION=v1.8.0
-CSI_PROVISIONER_VERSION=v3.5.0
-CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.8.0
+CSI_ATTACHER_VERSION=v4.4.0
+CSI_SNAPSHOTTER_VERSION=v6.3.0
+CSI_RESIZER_VERSION=v1.9.0
+CSI_PROVISIONER_VERSION=v3.6.0
+CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.9.0
 
 # e2e settings
 # - enable CEPH_CSI_RUN_ALL_TESTS when running tests with if it has root

charts/ceph-csi-cephfs/README.md

@@ -120,11 +120,11 @@ charts and their default values.
 | `sidecarLogLevel`                              | Set logging level for csi sidecar containers. Supported values from 0 to 5. 0 for general useful logs, 5 for trace level verbosity.                  | `1`                                                |
 | `nodeplugin.name`                              | Specifies the nodeplugin name                                                                                                                        | `nodeplugin`                                       |
 | `nodeplugin.updateStrategy`                    | Specifies the update Strategy. If you are using ceph-fuse client set this value to OnDelete                                                          | `RollingUpdate`                                    |
-| `nodeplugin.priorityClassName`                 | Set user created priorityclassName for csi plugin pods. default is system-node-critical which is highest priority                                    | `system-node-critical`                             |
+| `nodeplugin.priorityClassName`                 | Set user created priorityClassName for csi plugin pods. default is system-node-critical which is highest priority                                    | `system-node-critical`                             |
 | `nodeplugin.imagePullSecrets`                | Specifies imagePullSecrets for containers                                                                                                        | `[]`                                            |
 | `nodeplugin.profiling.enabled`                 | Specifies whether profiling should be enabled                                                                                                        | `false`                                            |
 | `nodeplugin.registrar.image.repository`        | Node-Registrar image repository URL                                                                                                                  | `registry.k8s.io/sig-storage/csi-node-driver-registrar` |
-| `nodeplugin.registrar.image.tag`               | Image tag                                                                                                                                            | `v2.8.0`                                           |
+| `nodeplugin.registrar.image.tag`               | Image tag                                                                                                                                            | `v2.9.0`                                           |
 | `nodeplugin.registrar.image.pullPolicy`        | Image pull policy                                                                                                                                    | `IfNotPresent`                                     |
 | `nodeplugin.plugin.image.repository`           | Nodeplugin image repository URL                                                                                                                      | `quay.io/cephcsi/cephcsi`                          |
 | `nodeplugin.plugin.image.tag`                  | Image tag                                                                                                                                            | `canary`                                           |
@@ -140,22 +140,22 @@ charts and their default values.
 | `provisioner.timeout`                          | GRPC timeout for waiting for creation or deletion of a volume                                                                                        | `60s`                                              |
 | `provisioner.clustername`                      | Cluster name to set on the subvolume                                                                                                                 | ""                                                 |
 | `provisioner.setmetadata`                      | Set metadata on volume                                                                                                                               | `true`                                             |
-| `provisioner.priorityClassName`                | Set user created priorityclassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical`                          |
+| `provisioner.priorityClassName`                | Set user created priorityClassName for csi provisioner pods. Default is `system-cluster-critical` which is less priority than `system-node-critical` | `system-cluster-critical`                          |
 | `provisioner.enableHostNetwork`                | Specifies whether hostNetwork is enabled for provisioner pod.                                                                                        | `false`                                            |
 | `provisioner.imagePullSecrets`                | Specifies imagePullSecrets for containers                                                                                                        | `[]`                                            |
 | `provisioner.profiling.enabled`                | Specifies whether profiling should be enabled                                                                                                        | `false`                                            |
 | `provisioner.provisioner.image.repository`     | Specifies the csi-provisioner image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-provisioner`      |
-| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v3.5.0`                                           |
+| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v3.6.0`                                           |
 | `provisioner.provisioner.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.provisioner.image.extraArgs`      | Specifies extra arguments for the provisioner sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.resizer.image.repository`         | Specifies the csi-resizer image repository URL                                                                                                       | `registry.k8s.io/sig-storage/csi-resizer`          |
-| `provisioner.resizer.image.tag`                | Specifies image tag                                                                                                                                  | `v1.8.0`                                           |
+| `provisioner.resizer.image.tag`                | Specifies image tag                                                                                                                                  | `v1.9.0`                                           |
 | `provisioner.resizer.image.pullPolicy`         | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.resizer.image.extraArgs`          | Specifies extra arguments for the resizer sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.resizer.name`                     | Specifies the name of csi-resizer sidecar                                                                                                            | `resizer`                                          |
 | `provisioner.resizer.enabled`                  | Specifies whether resizer sidecar is enabled                                                                                                         | `true`                                             |
 | `provisioner.snapshotter.image.repository`     | Specifies the csi-snapshotter image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-snapshotter`      |
-| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v6.2.2`                                           |
+| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v6.3.0`                                           |
 | `provisioner.snapshotter.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.snapshotter.image.extraArgs`      | Specifies extra arguments for the snapshotter sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.nodeSelector`                     | Specifies the node selector for provisioner deployment                                                                                               | `{}`                                               |

charts/ceph-csi-cephfs/values.yaml

@@ -92,7 +92,7 @@ nodeplugin:
   registrar:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
-      tag: v2.8.0
+      tag: v2.9.0
       pullPolicy: IfNotPresent
     resources: {}
 
@@ -184,7 +184,7 @@ provisioner:
   provisioner:
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
-      tag: v3.5.0
+      tag: v3.6.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -199,7 +199,7 @@ provisioner:
     enabled: true
     image:
       repository: gcr.io/k8s-staging-sig-storage/csi-resizer
-      tag: canary
+      tag: v1.9.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -209,7 +209,7 @@ provisioner:
   snapshotter:
     image:
       repository: registry.k8s.io/sig-storage/csi-snapshotter
-      tag: v6.2.2
+      tag: v6.3.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check

charts/ceph-csi-rbd/README.md

@@ -126,7 +126,7 @@ charts and their default values.
 | `nodeplugin.imagePullSecrets`                | Specifies imagePullSecrets for containers                                                                                                        | `[]`                                            |
 | `nodeplugin.profiling.enabled`                 | Specifies whether profiling should be enabled                                                                                                        | `false`                                            |
 | `nodeplugin.registrar.image.repository`        | Node Registrar image repository URL                                                                                                                  | `registry.k8s.io/sig-storage/csi-node-driver-registrar` |
-| `nodeplugin.registrar.image.tag`               | Image tag                                                                                                                                            | `v2.8.0`                                           |
+| `nodeplugin.registrar.image.tag`               | Image tag                                                                                                                                            | `v2.9.0`                                           |
 | `nodeplugin.registrar.image.pullPolicy`        | Image pull policy                                                                                                                                    | `IfNotPresent`                                     |
 | `nodeplugin.plugin.image.repository`           | Nodeplugin image repository URL                                                                                                                      | `quay.io/cephcsi/cephcsi`                          |
 | `nodeplugin.plugin.image.tag`                  | Image tag                                                                                                                                            | `canary`                                           |
@@ -151,23 +151,23 @@ charts and their default values.
 | `provisioner.imagePullSecrets`                | Specifies imagePullSecrets for containers                                                                                                        | `[]`                                            |
 | `provisioner.profiling.enabled`                | Specifies whether profiling should be enabled                                                                                                        | `false`                                            |
 | `provisioner.provisioner.image.repository`     | Specifies the csi-provisioner image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-provisioner`           |
-| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v3.5.0`                                           |
+| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v3.6.0`                                           |
 | `provisioner.provisioner.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.provisioner.image.extraArgs`      | Specifies extra arguments for the provisioner sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.attacher.image.repository`        | Specifies the csi-attacher image repository URL                                                                                                      | `registry.k8s.io/sig-storage/csi-attacher`              |
-| `provisioner.attacher.image.tag`               | Specifies image tag                                                                                                                                  | `v4.3.0`                                           |
+| `provisioner.attacher.image.tag`               | Specifies image tag                                                                                                                                  | `v4.4.0`                                           |
 | `provisioner.attacher.image.pullPolicy`        | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.attacher.image.extraArgs`         | Specifies extra arguments for the attacher sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.attacher.name`                    | Specifies the name of csi-attacher sidecar                                                                                                           | `attacher`                                         |
 | `provisioner.attacher.enabled`                 | Specifies whether attacher sidecar is enabled                                                                                                        | `true`                                             |
 | `provisioner.resizer.image.repository`         | Specifies the csi-resizer image repository URL                                                                                                       | `registry.k8s.io/sig-storage/csi-resizer`               |
-| `provisioner.resizer.image.tag`                | Specifies image tag                                                                                                                                  | `v1.8.0`                                           |
+| `provisioner.resizer.image.tag`                | Specifies image tag                                                                                                                                  | `v1.9.0`                                           |
 | `provisioner.resizer.image.pullPolicy`         | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.resizer.image.extraArgs`          | Specifies extra arguments for the resizer sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.resizer.name`                     | Specifies the name of csi-resizer sidecar                                                                                                            | `resizer`                                          |
 | `provisioner.resizer.enabled`                  | Specifies whether resizer sidecar is enabled                                                                                                         | `true`                                             |
 | `provisioner.snapshotter.image.repository`     | Specifies the csi-snapshotter image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-snapshotter`          |
-| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v6.2.2`                                           |
+| `provisioner.snapshotter.image.tag`            | Specifies image tag                                                                                                                                  | `v6.3.0`                                           |
 | `provisioner.snapshotter.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.snapshotter.image.extraArgs`      | Specifies extra arguments for the snapshotter sidecar                                                                                                                                | `[]`                                     |
 | `provisioner.nodeSelector`                     | Specifies the node selector for provisioner deployment                                                                                               | `{}`                                               |
@@ -176,6 +176,8 @@ charts and their default values.
 | `provisioner.podSecurityPolicy.enabled`        | Specifies whether podSecurityPolicy is enabled                                                                                                       | `false`                                            |
 | `topology.enabled`                             | Specifies whether topology based provisioning support should be exposed by CSI                                                                       | `false`                                            |
 | `topology.domainLabels`                        | DomainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains                                               | `{}`                                               |
+| `readAffinity.enabled` | Enable read affinity for RBD volumes. Recommended to set to true if running kernel 5.8 or newer. | `false` |
+| `readAffinity.crushLocationLabels` | Define which node labels to use as CRUSH location. This should correspond to the values set in the CRUSH map. For more information, click [here](https://github.com/ceph/ceph-csi/blob/v3.9.0/docs/deploy-rbd.md#read-affinity-using-crush-locations-for-rbd-volumes)| `[]` |
 | `provisionerSocketFile`                        | The filename of the provisioner socket                                                                                                               | `csi-provisioner.sock`                             |
 | `pluginSocketFile`                             | The filename of the plugin socket                                                                                                                    | `csi.sock`                                         |
 | `kubeletDir`                                   | kubelet working directory                                                                                                                            | `/var/lib/kubelet`                                 |

charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml

@@ -86,6 +86,10 @@ spec:
 {{- end }}
 {{- if .Values.nodeplugin.profiling.enabled }}
             - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
+{{- end }}
+            - "--enable-read-affinity={{ and .Values.readAffinity .Values.readAffinity.enabled }}"
+{{- if and .Values.readAffinity .Values.readAffinity.enabled }}
+            - "--crush-location-labels={{ .Values.readAffinity.crushLocationLabels | join "," }}"
 {{- end }}
           env:
             - name: POD_IP

charts/ceph-csi-rbd/values.yaml

@@ -115,7 +115,7 @@ nodeplugin:
   registrar:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
-      tag: v2.8.0
+      tag: v2.9.0
       pullPolicy: IfNotPresent
     resources: {}
 
@@ -217,7 +217,7 @@ provisioner:
   provisioner:
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
-      tag: v3.5.0
+      tag: v3.6.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -232,7 +232,7 @@ provisioner:
     enabled: true
     image:
       repository: registry.k8s.io/sig-storage/csi-attacher
-      tag: v4.3.0
+      tag: v4.4.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -244,7 +244,7 @@ provisioner:
     enabled: true
     image:
       repository: gcr.io/k8s-staging-sig-storage/csi-resizer
-      tag: canary
+      tag: v1.9.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -254,7 +254,7 @@ provisioner:
   snapshotter:
     image:
       repository: registry.k8s.io/sig-storage/csi-snapshotter
-      tag: v6.2.2
+      tag: v6.3.0
       pullPolicy: IfNotPresent
     resources: {}
     ## For further options, check
@@ -279,6 +279,17 @@ topology:
     - failure-domain/region
     - failure-domain/zone
 
+# readAffinity:
+# Enable read affinity for RBD volumes. Recommended to
+# set to true if running kernel 5.8 or newer.
+# enabled: false
+# Define which node labels to use as CRUSH location.
+# This should correspond to the values set in the CRUSH map.
+# NOTE: the value here serves as an example
+# crushLocationLabels:
+#   - topology.kubernetes.io/region
+#   - topology.kubernetes.io/zone
+
 storageClass:
   # Specifies whether the storageclass should be created
   create: false

cmd/cephcsi.go

@@ -232,7 +232,7 @@ func main() {
 	switch conf.Vtype {
 	case rbdType:
 		validateCloneDepthFlag(&conf)
-		validateMaxSnaphostFlag(&conf)
+		validateMaxSnapshotFlag(&conf)
 		driver := rbddriver.NewDriver()
 		driver.Run(&conf)
 
@@ -304,7 +304,7 @@ func validateCloneDepthFlag(conf *util.Config) {
 	}
 }
 
-func validateMaxSnaphostFlag(conf *util.Config) {
+func validateMaxSnapshotFlag(conf *util.Config) {
 	// maximum number of snapshots on an image are 510 [1] and 16 images in
 	// a parent/child chain [2],keeping snapshot limit to 500 to avoid issues.
 	// [1] https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c#L98

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -43,7 +43,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+          image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -62,7 +62,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-resizer
-          image: gcr.io/k8s-staging-sig-storage/csi-resizer:canary
+          image: gcr.io/k8s-staging-sig-storage/csi-resizer:v1.9.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -79,7 +79,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

deploy/cephfs/kubernetes/csi-cephfsplugin.yaml

@@ -27,7 +27,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
           args:
             - "--v=1"
             - "--csi-address=/csi/csi.sock"

deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml

@@ -40,7 +40,7 @@ spec:
               topologyKey: "kubernetes.io/hostname"
       containers:
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+          image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -57,7 +57,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-resizer
-          image: gcr.io/k8s-staging-sig-storage/csi-resizer:canary
+          image: gcr.io/k8s-staging-sig-storage/csi-resizer:v1.9.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -73,7 +73,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

deploy/nfs/kubernetes/csi-nfsplugin.yaml

@@ -27,7 +27,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
           args:
             - "--v=1"
             - "--csi-address=/csi/csi.sock"

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -47,7 +47,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v3.5.0
+          image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -69,7 +69,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.2.2
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
@@ -84,7 +84,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-attacher
-          image: registry.k8s.io/sig-storage/csi-attacher:v4.3.0
+          image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0
           args:
             - "--v=1"
             - "--csi-address=$(ADDRESS)"
@@ -99,7 +99,7 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-resizer
-          image: gcr.io/k8s-staging-sig-storage/csi-resizer:canary
+          image: gcr.io/k8s-staging-sig-storage/csi-resizer:v1.9.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -29,7 +29,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.8.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
           args:
             - "--v=1"
             - "--csi-address=/csi/csi.sock"

docs/ceph-csi-upgrade.md

@@ -3,7 +3,6 @@
 - [Ceph-csi Upgrade](#ceph-csi-upgrade)
    - [Pre-upgrade considerations](#pre-upgrade-considerations)
       - [Snapshot-controller and snapshot crd](#snapshot-controller-and-snapshot-crd)
-         - [Snapshot API version support matrix](#snapshot-api-version-support-matrix)
    - [Upgrading from previous releases](#upgrading-from-previous-releases)
    - [Upgrading from v3.8 to v3.9](#upgrading-from-v38-to-v39)
       - [Upgrading CephFS](#upgrading-cephfs)
@@ -14,8 +13,6 @@
             - [2.1 Update the CephFS Nodeplugin RBAC](#21-update-the-cephfs-nodeplugin-rbac)
             - [2.2 Update the CephFS Nodeplugin daemonset](#22-update-the-cephfs-nodeplugin-daemonset)
             - [2.3 Manual deletion of CephFS Nodeplugin daemonset pods](#23-manual-deletion-of-cephfs-nodeplugin-daemonset-pods)
-            - [2.4 Modifying MountOptions in Storageclass and PersistentVolumes](#24-modifying-mountoptions-in-storageclass-and-persistentvolumes)
-         - [Delete removed CephFS PSP, Role and RoleBinding](#delete-removed-cephfs-psp-role-and-rolebinding)
       - [Upgrading RBD](#upgrading-rbd)
          - [3. Upgrade RBD Provisioner resources](#3-upgrade-rbd-provisioner-resources)
             - [3.1 Update the RBD Provisioner RBAC](#31-update-the-rbd-provisioner-rbac)
@@ -23,7 +20,6 @@
          - [4. Upgrade RBD Nodeplugin resources](#4-upgrade-rbd-nodeplugin-resources)
             - [4.1 Update the RBD Nodeplugin RBAC](#41-update-the-rbd-nodeplugin-rbac)
             - [4.2 Update the RBD Nodeplugin daemonset](#42-update-the-rbd-nodeplugin-daemonset)
-         - [Delete removed RBD PSP, Role and RoleBinding](#delete-removed-rbd-psp-role-and-rolebinding)
       - [Upgrading NFS](#upgrading-nfs)
          - [5. Upgrade NFS Provisioner resources](#5-upgrade-nfs-provisioner-resources)
             - [5.1 Update the NFS Provisioner RBAC](#51-update-the-nfs-provisioner-rbac)
@@ -61,13 +57,6 @@ Its kubernetes distributor responsibility to install new snapshot
 controller and snapshot CRD. more info can be found
 [here](https://github.com/kubernetes-csi/external-snapshotter/tree/master#usage)
 
-#### Snapshot API version support matrix
-
-| Snapshot API version | Kubernetes Version   | Snapshot-Controller + CRDs Version | Sidecar Version |
-| -------------------- | -------------------- | ---------------------------------- | --------------- |
-| v1beta1              | v1.17 =< k8s < v1.20 | v2.x =< snapshot-controller < v4.x | sidecar >= v2.x |
-| v1                   | k8s >= v1.20         | snapshot-controller >= v4.x        | sidecar >= v2.x |
-
 **Note:** We recommend to use {sidecar, controller, crds} of same version
 
 ## Upgrading from previous releases
@@ -116,9 +105,6 @@ Warning: kubectl apply should be used on resource created by either kubectl crea
 
 ### Upgrading CephFS
 
-If existing cephfs storageclasses' `MountOptions` are set, please refer to
-[modifying mount options](#24-modifying-mountoptions-in-storageclass-and-persistentvolumes)
-section.
 Upgrading cephfs csi includes upgrade of cephfs driver and as well as
 kubernetes sidecar containers and also the permissions required for the
 kubernetes sidecar containers, lets upgrade the things one by one
@@ -230,45 +216,6 @@ For each node:
    - The pod deletion causes the pods to be restarted and updated automatically
      on the node.
 
-##### 2.4 Modifying MountOptions in Storageclass and PersistentVolumes
-
-CephCSI, starting from release v3.9.0, will pass the options specified in the
-StorageClass's `MountOptions` during both `NodeStageVolume` (kernel cephfs or
-ceph-fuse mount operation) and `NodePublishVolume` (bind mount) operations.
-Therefore, only common options that is acceptable during both the above
-described operations needs to be set in StorageClass's `MountOptions`.
-If invalid mount options are set in StorageClass's `MountOptions`
-such as `"debug"`, the mounting of cephFS PVCs will fail.
-
-Follow the below steps to update the StorageClass's `MountOptions`:
-
-- Take a backup of the StorageClass using
-  `kubectl get sc <storageclass-name> -o yaml > sc.yaml`.
-- Edit `sc.yaml` to remove the invalid mount options from `MountOptions` field.
-- Delete the StorageClass using `kubectl delete sc <storageclass-name>`.
-- Recreate the StorageClass using `kubectl create -f sc.yaml`.
-
-Follow the below steps to update the PersistentVolume's `MountOptions`:
-
-- Identify cephFS PersistentVolumes using
-  `kubectl get pv | grep <storageclass-name>`.
-- and remove invalid mount options from `MountOptions` field
-  in the PersistentVolume's using `kubectl edit pv <pv-name>`.
-
-#### Delete removed CephFS PSP, Role and RoleBinding
-
-As PSP is deprecated in Kubernetes v1.21.0. Delete PSP related objects as PSP
-support for CephFS is removed.
-
-```console
-kubectl delete psp cephfs-csi-provisioner-psp --ignore-not-found
-kubectl delete role cephfs-csi-provisioner-psp --ignore-not-found
-kubectl delete rolebinding cephfs-csi-provisioner-psp --ignore-not-found
-kubectl delete psp cephfs-csi-nodeplugin-psp --ignore-not-found
-kubectl delete role cephfs-csi-nodeplugin-psp --ignore-not-found
-kubectl delete rolebinding cephfs-csi-nodeplugin-psp --ignore-not-found
-```
-
 we have successfully upgraded cephfs csi from v3.8 to v3.9
 
 ### Upgrading RBD
@@ -288,7 +235,6 @@ Provisioner deployment
 $ kubectl apply -f deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
 serviceaccount/rbd-csi-provisioner configured
 clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner configured
-clusterrole.rbac.authorization.k8s.io/rbd-external-provisioner-runner-rules configured
 clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role configured
 role.rbac.authorization.k8s.io/rbd-external-provisioner-cfg configured
 rolebinding.rbac.authorization.k8s.io/rbd-csi-provisioner-role-cfg configured
@@ -323,7 +269,6 @@ nodeplugin daemonset
 $ kubectl apply -f deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
 serviceaccount/rbd-csi-nodeplugin configured
 clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin configured
-clusterrole.rbac.authorization.k8s.io/rbd-csi-nodeplugin-rules configured
 clusterrolebinding.rbac.authorization.k8s.io/rbd-csi-nodeplugin configured
 ```
 
@@ -335,23 +280,6 @@ daemonset.apps/csi-rbdplugin configured
 service/csi-metrics-rbdplugin configured
 ```
 
-#### Delete removed RBD PSP, Role and RoleBinding
-
-As PSP is deprecated in Kubernetes v1.21.0. Delete PSP related objects as PSP
-support for RBD is removed.
-
-```console
-kubectl delete psp rbd-csi-provisioner-psp --ignore-not-found
-kubectl delete role rbd-csi-provisioner-psp --ignore-not-found
-kubectl delete rolebinding rbd-csi-provisioner-psp --ignore-not-found
-kubectl delete psp rbd-csi-nodeplugin-psp --ignore-not-found
-kubectl delete role rbd-csi-nodeplugin-psp --ignore-not-found
-kubectl delete rolebinding rbd-csi-nodeplugin-psp --ignore-not-found
-kubectl delete psp rbd-csi-vault-token-review-psp --ignore-not-found
-kubectl delete role rbd-csi-vault-token-review-psp --ignore-not-found
-kubectl delete rolebinding rbd-csi-vault-token-review-psp --ignore-not-found
-```
-
 we have successfully upgraded RBD csi from v3.8 to v3.9
 
 ### Upgrading NFS

docs/cephfs-snapshot-backed-volumes.md

@@ -18,17 +18,18 @@ information.
 ### Provisioning a snapshot-backed volume from a volume snapshot
 
 For provisioning new snapshot-backed volumes, following configuration must be
-set for storage class(es) and their PVCs respectively:
+set for PVCs:
 
-* StorageClass:
-   * Specify `backingSnapshot: "true"` parameter.
 * PersistentVolumeClaim:
-   * Set `storageClassName` to point to your storage class with backing
-    snapshots enabled.
+   * Set `storageClassName` to point to your existing cephFS storage class.
    * Define `spec.dataSource` for your desired source volume snapshot.
    * Set `spec.accessModes` to `ReadOnlyMany`. This is the only access mode that
     is supported by this feature.
 
+Note:- We can also disable shallowVolume by setting `backingSnapshot: "false"
+in cephFS storageclass. If the value is set in the storageclass when requested
+for `ReadOnlyMany` PVC a clone will get created in ceph cluster.
+
 ### Mounting snapshots from pre-provisioned volumes
 
 Steps for defining a PersistentVolume and PersistentVolumeClaim for

docs/deploy-cephfs.md

@@ -151,6 +151,11 @@ for more information.
 kubectl create -f ../../ceph-conf.yaml
 ```
 
+**Deploy prerequisites for CSI Snapshot:**
+
+If you intend to use the snapshot functionality in Kubernetes cluster,
+please refer to [snap-clone.md](./snap-clone.md#prerequisite)
+
 **Deploy CSI sidecar containers:**
 
 ```bash

docs/deploy-rbd.md

@@ -137,6 +137,11 @@ for more information.
 kubectl create -f ../../ceph-conf.yaml
 ```
 
+**Deploy prerequisites for CSI Snapshot:**
+
+If you intend to use the snapshot functionality in Kubernetes cluster,
+please refer to [snap-clone.md](./snap-clone.md#prerequisite)
+
 **Deploy CSI sidecar containers:**
 
 ```bash

docs/design/proposals/rbd-qos.md

@@ -0,0 +1,271 @@
+# Design Doc for RBD QoS using cgroup v2
+
+## Introduction
+
+The RBD QoS (Quality of Service) design aims to address the issue of IO noisy
+neighbor problems encountered in early Ceph deployments catering to OpenStack
+environments. These problems were effectively managed by implementing QEMU
+throttling at the virtio-blk/scsi level. To further enhance this,
+capacity-based IOPS were introduced, providing a more dynamic experience
+similar to public cloud environments.
+
+The challenge arises in virtual environments, where a noisy neighbor can lead
+to performance degradation for other instances sharing the same resources.
+Although it's uncommon to observe noisy neighbor issues in Kubernetes
+environments backed by Ceph storage, the possibility exists. The existing QoS
+support with rbd-nbd doesn't apply to krbd, and as rbd-nbd isn't suitable for
+container production workloads, a solution is needed for krbd.
+
+To mitigate resource starvation issues, setting QoS at the device level through
+cgroup v2 when enabled becomes crucial. This approach guarantees that I/O
+capacity isn't overcommitted and is fairly distributed among workloads.
+
+## Dependency
+
+* cgroup v2 must be enabled on the Node
+* We might have Kubernetes dependency as well
+* Container runtime dependency that supports cgroupv2
+
+## Manual steps for implementing RBD QoS in a Kubernetes Cluster
+
+```bash
+[$] ssh root@node1
+sh-4.4# chroot /host
+sh-5.1# cat /proc/partitions
+major minor  #blocks  name
+
+ 259        0  125829120 nvme0n1
+ 259        1       1024 nvme0n1p1
+ 259        2     130048 nvme0n1p2
+ 259        3     393216 nvme0n1p3
+ 259        4  125303791 nvme0n1p4
+ 259        6   52428800 nvme2n1
+   7        0  536870912 loop0
+ 259        5  536870912 nvme1n1
+ 252        0   52428800 rbd0
+sh-5.1#
+```
+
+Once the rbd device is mapped on the node we get the device's major and minor
+number we need to set the io limit on the device but we need to find the right
+cgroup file where we need to set the limit
+
+Kubernetes/Openshift creates a custom cgroup hierarchy for the pods it created
+but start is `/sys/fs/cgroup`  folder
+
+```bash
+sh-5.1# cd /sys/fs/cgroup/
+sh-5.1# ls
+cgroup.controllers cgroup.subtree_control cpuset.mems.effective  io.stat   memory.reclaim   sys-kernel-debug.mount
+cgroup.max.depth cgroup.threads  dev-hugepages.mount    kubepods.slice  memory.stat   sys-kernel-tracing.mount
+cgroup.max.descendants cpu.pressure  dev-mqueue.mount       machine.slice  misc.capacity   system.slice
+cgroup.procs  cpu.stat  init.scope        memory.numa_stat  sys-fs-fuse-connections.mount user.slice
+cgroup.stat  cpuset.cpus.effective io.pressure        memory.pressure  sys-kernel-config.mount
+```
+
+`kubepods.slice` is the starting point and it contains multiple slices
+
+```bash
+sh-5.1# cd kubepods.slice
+sh-5.1# ls
+cgroup.controllers cpuset.cpus    hugetlb.2MB.rsvd.max       memory.pressure
+cgroup.events  cpuset.cpus.effective   io.bfq.weight        memory.reclaim
+cgroup.freeze  cpuset.cpus.partition   io.latency        memory.stat
+cgroup.kill  cpuset.mems    io.max        memory.swap.current
+cgroup.max.depth cpuset.mems.effective   io.pressure        memory.swap.events
+cgroup.max.descendants hugetlb.1GB.current   io.stat        memory.swap.high
+cgroup.procs  hugetlb.1GB.events   kubepods-besteffort.slice      memory.swap.max
+cgroup.stat  hugetlb.1GB.events.local  kubepods-burstable.slice      memory.zswap.current
+cgroup.subtree_control hugetlb.1GB.max    kubepods-pod2b38830b_c2d6_4528_8935_b1c08511b1e3.slice  memory.zswap.max
+cgroup.threads  hugetlb.1GB.numa_stat   memory.current       misc.current
+cgroup.type  hugetlb.1GB.rsvd.current  memory.events        misc.max
+cpu.idle  hugetlb.1GB.rsvd.max   memory.events.local       pids.current
+cpu.max   hugetlb.2MB.current   memory.high        pids.events
+cpu.max.burst  hugetlb.2MB.events   memory.low        pids.max
+cpu.pressure  hugetlb.2MB.events.local  memory.max        rdma.current
+cpu.stat  hugetlb.2MB.max    memory.min        rdma.max
+cpu.weight  hugetlb.2MB.numa_stat   memory.numa_stat
+cpu.weight.nice  hugetlb.2MB.rsvd.current  memory.oom.group
+```
+
+Based on the QoS of the pod, either our application pod will end up in the
+above `kubepods-besteffort.slice` or `kubepods-burstable.slice` or
+`kubepods.slice` (Guaranteed QoS) cgroup. The 3 QoS classes are defined
+[here](https://kubernetes.io/docs/concepts/workloads/pods/pod-QoS/#quality-of-service-classes)
+
+To identify the right cgroup file, we need pod UUID and container UUID from the
+`pod yaml` output
+
+```bash
+[$]kubectl get po csi-rbd-demo-pod -oyaml |grep uid
+  uid: cdf7b785-4eb7-44f7-99cc-ef53890f4dfd
+[$]kubectl get po csi-rbd-demo-pod -oyaml |grep -i containerID
+  - containerID: cri-o://77e57fbbc0f0630f41f9f154f4b5fe368b6dcf7bef7dcd75a9c4b56676f10bc9
+[$]kubectl get po csi-rbd-demo-pod -oyaml |grep -i qosClass
+  qosClass: BestEffort
+```
+
+Now check in the `kubepods-besteffort.slice` and identify the right path using
+pod UID and container UID
+
+Before that check `io.max` on the application pod and see if there is any limit
+
+```bash
+[$]kubectl exec -it csi-rbd-demo-pod -- sh
+sh-4.4# cat /sys/fs/cgroup/io.max
+sh-4.4#
+```
+
+Come back to the Node and find the right cgroup scope
+
+```bash
+sh-5.1# cd kubepods-besteffort.slice/kubepods-besteffort-podcdf7b785_4eb7_44f7_99cc_ef53890f4dfd.slice/crio-77e57fbbc0f0630f41f9f154f4b5fe368b6dcf7bef7dcd75a9c4b56676f10bc9.scope/
+
+
+sh-5.1# echo "252:0 wbps=1048576" > io.max
+sh-5.1# cat io.max
+252:0 rbps=max wbps=1048576 riops=max wiops=max
+```
+
+Now go back to the application pod and check if we have the right limit set
+
+```bash
+[$]kubectl exec -it csi-rbd-demo-pod -- sh
+sh-4.4# cat /sys/fs/cgroup/io.max
+252:0 rbps=max wbps=1048576 riops=max wiops=max
+sh-4.4#
+```
+
+Note:- We can only support the QoS that cgroup v2 io controller supports, this
+means that cumulative read+write QoS limits won't be supported.
+
+Below are the configurations that will be supported
+
+|  Parameter     |  Description     |
+|  ---  |  ---  |
+|  MaxReadIOPS     | Max read IO operations per second      |
+|  MaxWriteIOPS     | Max write IO operations per second      |
+|  MaxReadBytesPerSecond     |  Max read bytes per second     |
+|  MaxWriteBytesPerSecond     |  Max write bytes per second     |
+
+## Different approaches
+
+The above solution can be implemented using 3 different approaches.
+
+### 1. QoS using new parameters in RBD StorageClass
+
+```yaml
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+   name: csi-rbd-sc
+provisioner: rbd.csi.ceph.com
+parameters:
+  MaxReadIOPS: ""
+  MaxWriteIOPS: ""
+  MaxReadBytesPerSecond: ""
+  MaxWriteBytesPerSecond: ""
+```
+
+#### Implementation for StorageClass QoS
+
+1. Create new storageClass with new parameters for QoS
+1. Modify CSIDriver object to pass pod details to the NodePublishVolume CSI
+   procedure
+1. During NodePublishVolume CSI procedure
+   * Retrieve the QoS configuration from the volumeContext in NodePublishRequest
+   * Identify the rbd device using the NodeStageVolumePath
+   * Get the pod UUID from the NodeStageVolume
+   * Set io.max file in all the containers in the pod
+
+#### Drawbacks of StorageClass QoS
+
+1. No way to update the QoS at runtime
+1. Need to take a backup and restore to New QoS StorageClass
+1. Delete and Recreate the PV object
+
+### 2. QoS using parameters in VolumeAttributeClass
+
+```yaml
+apiVersion: storage.k8s.io/v1alpha1
+kind: VolumeAttributesClass
+metadata:
+  name: silver
+parameters:
+  MaxReadIOPS: ""
+  MaxWriteIOPS: ""
+  MaxReadBytesPerSecond: ""
+  MaxWriteBytesPerSecond: ""
+```
+
+VolumeAttributesClassName is a new parameter in the PVC object the user can
+choose from and this can also be updated or removed later.
+
+This new VolumeAttributeClass is designed to keep storage that supports setting
+QoS at the storage level which means setting some configuration at the storage
+(like QoS for nbd)
+
+#### Implementation of VolumeAttributeClass QoS
+
+1. Modify CSIDriver object to pass pod details to the NodePublishVolume CSI
+   procedure
+1. Add support in Ceph-CSI to expose ModifyVolume CSI procedure
+1. Ceph-CSI will store QoS in the rbd image metadata
+1. During NodeStage operation retrieve the image metadata and store it in
+   stagingPath
+1. Whenever a new pod comes in apply the QoS
+
+#### Drawbacks of VolumeAttributeClass QoS
+
+One problem with above is all application need to be scaled downed and scaled
+up to get the new QoS value even though its changed in the PVC object, this is
+sometime impossible as it will have downtime.
+
+### 3. QoS using parameters in VolumeAttributeClass with NodePublish Secret
+
+1. Modify CSIDriver object to pass pod details to the NodePublishVolume CSI
+   procedure
+1. Add support in Ceph-CSI to expose ModifyVolume CSI procedure
+1. Ceph-CSI will store QoS in the rbd image metadata
+1. During NodePublishVolume operation retrieve the QoS from image metadata
+1. Whenever a new pod comes in apply the QoS
+
+This solution addresses the aforementioned issue, but it requires a secret to
+communicate with the ceph cluster. Therefore, we must create a new
+PublishSecret for the storageClass, which may be beneficial when Kubernetes
+eventually enables Node operations.
+
+Both options 2 and 3 are contingent upon changes to the CSI spec and Kubernetes
+support. Additionally,
+[VolumeAttributeClass](https://github.com/kubernetes/enhancements/blob/master/keps/sig-storage/3751-volume-attributes-class/README.md)
+is currently being developed within the Kubernetes realm and will initially be
+in the Alpha stage. Consequently, it will be disabled by default.
+
+#### Advantages of QoS using VolumeAttributeClass
+
+1. No Restore/Clone operation is required to change the QoS
+1. Easily QoS can be changed for existing PVC only with second approach not
+   with third as it needs new secret.
+
+### Hybrid Approach
+
+Considering the advantages and drawbacks, we can use StorageClass and
+VolumeAttributeClass to support QoS, with VolumeAttributeClass taking
+precedence over StorageClass. This approach offers a flexible solution that
+accounts for dynamic changes while addressing the challenges of existing
+approaches.
+
+### References
+
+Some of the useful links that helped me to understand cgroup v2 and how to set
+QoS on the device.
+
+* [Kubernetes cgroup v2
+  Architecture](https://kubernetes.io/docs/concepts/architecture/cgroups/)
+* [cgroup v2 kernel doc](https://docs.kernel.org/admin-guide/cgroup-v2.html)
+* [ceph RBD QoS tracker](https://tracker.ceph.com/issues/36191)
+* [cgroup v2 io
+  controller](https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html)
+* [Kubernetes IOPS
+  issue](https://github.com/kubernetes/kubernetes/issues/92287)

docs/static-pvc.md

@@ -257,6 +257,7 @@ spec:
   resources:
     requests:
       storage: 1Gi
+  storageClassName: ""
   volumeMode: Filesystem
   # volumeName should be same as PV name
   volumeName: cephfs-static-pv

e2e/deployment.go

@@ -231,15 +231,19 @@ func (yr *yamlResource) Do(action kubectlAction) error {
 // replaceNamespaceInTemplate() on it. There are several options for adjusting
 // templates, each has their own comment.
 type yamlResourceNamespaced struct {
-	filename  string
-	namespace string
+	filename            string
+	namespace           string
+	domainLabel         string
+	crushLocationLabels string
 
 	// set the number of replicas in a Deployment to 1.
 	oneReplica bool
 
 	// enable topology support (for RBD)
 	enableTopology bool
-	domainLabel    string
+
+	// enable read affinity support (for RBD)
+	enableReadAffinity bool
 }
 
 func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error {
@@ -260,6 +264,14 @@ func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error {
 		data = addTopologyDomainsToDSYaml(data, yrn.domainLabel)
 	}
 
+	if yrn.enableReadAffinity {
+		data = enableReadAffinityInTemplate(data)
+	}
+
+	if yrn.crushLocationLabels != "" {
+		data = addCrsuhLocationLabels(data, yrn.crushLocationLabels)
+	}
+
 	err = retryKubectlInput(yrn.namespace, action, data, deployTimeout)
 	if err != nil {
 		return fmt.Errorf("failed to %s resource %q in namespace %q: %w", action, yrn.filename, yrn.namespace, err)

e2e/pod.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"strings"
 	"time"
 
@@ -623,3 +624,102 @@ func verifySeLinuxMountOption(
 
 	return nil
 }
+
+// verifyReadAffinity verifies if read affinity is enabled by checking if read_from_replica
+// and crush_location options are present in the device config file (/sys/devices/rbd/0/config_info).
+func verifyReadAffinity(
+	f *framework.Framework,
+	pvcPath, appPath, daemonSetName, cn, ns string,
+) error {
+	readFromReplicaOption := "read_from_replica=localize"
+	expectedCrushLocationValues := map[string]string{
+		strings.Split(crushLocationRegionLabel, "/")[1]: crushLocationRegionValue,
+		strings.Split(crushLocationZoneLabel, "/")[1]:   crushLocationZoneValue,
+	}
+
+	// create PVC
+	pvc, err := loadPVC(pvcPath)
+	if err != nil {
+		return fmt.Errorf("failed to load pvc: %w", err)
+	}
+	pvc.Namespace = f.UniqueName
+	err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("failed to create PVC: %w", err)
+	}
+	app, err := loadApp(appPath)
+	if err != nil {
+		return fmt.Errorf("failed to load application: %w", err)
+	}
+	app.Namespace = f.UniqueName
+	err = createApp(f.ClientSet, app, deployTimeout)
+	if err != nil {
+		return fmt.Errorf("failed to create application: %w", err)
+	}
+
+	imageInfo, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
+	if err != nil {
+		return fmt.Errorf("failed to get imageInfo: %w", err)
+	}
+
+	selector, err := getDaemonSetLabelSelector(f, ns, daemonSetName)
+	if err != nil {
+		return fmt.Errorf("failed to get selector label %w", err)
+	}
+
+	opt := metav1.ListOptions{
+		LabelSelector: selector,
+	}
+
+	command := "cat /sys/devices/rbd/*/config_info"
+	configInfos, _, err := execCommandInContainer(f, command, ns, cn, &opt)
+	if err != nil {
+		return fmt.Errorf("failed to execute command %s: %w", command, err)
+	}
+
+	var configInfo string
+	for _, config := range strings.Split(configInfos, "\n") {
+		if config == "" || !strings.Contains(config, imageInfo.imageName) {
+			continue
+		}
+		configInfo = config
+
+		break
+	}
+
+	if configInfo == "" {
+		return errors.New("failed to get config_info file")
+	}
+
+	if !strings.Contains(configInfo, readFromReplicaOption) {
+		return fmt.Errorf("option %s not found in config_info: %s", readFromReplicaOption, configInfo)
+	}
+
+	crushLocationPattern := "crush_location=([^,]+)"
+	regex := regexp.MustCompile(crushLocationPattern)
+	match := regex.FindString(configInfo)
+	if match == "" {
+		return fmt.Errorf("option crush_location not found in config_info: %s", configInfo)
+	}
+
+	crushLocationValue := strings.Split(match, "=")[1]
+	keyValues := strings.Split(crushLocationValue, "|")
+	actualCrushLocationValues := make(map[string]string)
+
+	for _, keyValue := range keyValues {
+		s := strings.Split(keyValue, ":")
+		actualCrushLocationValues[s[0]] = s[1]
+	}
+	for key, expectedValue := range expectedCrushLocationValues {
+		if actualValue, exists := actualCrushLocationValues[key]; !(exists && actualValue == expectedValue) {
+			return fmt.Errorf("crush location %s:%s not found in config_info : %s", key, expectedValue, configInfo)
+		}
+	}
+
+	err = deletePVCAndApp("", f, pvc, app)
+	if err != nil {
+		return fmt.Errorf("failed to delete PVC and application: %w", err)
+	}
+
+	return nil
+}

e2e/rbd.go

@@ -65,6 +65,12 @@ var (
 	rbdTopologyPool     = "newrbdpool"
 	rbdTopologyDataPool = "replicapool" // NOTE: should be different than rbdTopologyPool for test to be effective
 
+	// CRUSH location node labels & values.
+	crushLocationRegionLabel = "topology.kubernetes.io/region"
+	crushLocationRegionValue = "east"
+	crushLocationZoneLabel   = "topology.kubernetes.io/zone"
+	crushLocationZoneValue   = "east-zone1"
+
 	// yaml files required for deployment.
 	pvcPath                = rbdExamplePath + "pvc.yaml"
 	appPath                = rbdExamplePath + "pod.yaml"
@@ -161,9 +167,11 @@ func createORDeleteRbdResources(action kubectlAction) {
 		},
 		// the node-plugin itself
 		&yamlResourceNamespaced{
-			filename:    rbdDirPath + rbdNodePlugin,
-			namespace:   cephCSINamespace,
-			domainLabel: nodeRegionLabel + "," + nodeZoneLabel,
+			filename:            rbdDirPath + rbdNodePlugin,
+			namespace:           cephCSINamespace,
+			domainLabel:         nodeRegionLabel + "," + nodeZoneLabel,
+			enableReadAffinity:  true,
+			crushLocationLabels: crushLocationRegionLabel + "," + crushLocationZoneLabel,
 		},
 	}
 
@@ -275,6 +283,14 @@ var _ = Describe("RBD", func() {
 			if err != nil {
 				framework.Failf("failed to create node label: %v", err)
 			}
+			err = createNodeLabel(f, crushLocationRegionLabel, crushLocationRegionValue)
+			if err != nil {
+				framework.Failf("failed to create node label: %v", err)
+			}
+			err = createNodeLabel(f, crushLocationZoneLabel, crushLocationZoneValue)
+			if err != nil {
+				framework.Failf("failed to create node label: %v", err)
+			}
 			if cephCSINamespace != defaultNs {
 				err = createNamespace(c, cephCSINamespace)
 				if err != nil {
@@ -409,6 +425,15 @@ var _ = Describe("RBD", func() {
 		if err != nil {
 			framework.Failf("failed to delete node label: %v", err)
 		}
+		// Remove the CRUSH Location labels
+		err = deleteNodeLabel(c, crushLocationRegionLabel)
+		if err != nil {
+			framework.Failf("failed to delete node label: %v", err)
+		}
+		err = deleteNodeLabel(c, crushLocationZoneLabel)
+		if err != nil {
+			framework.Failf("failed to delete node label: %v", err)
+		}
 	})
 
 	Context("Test RBD CSI", func() {
@@ -444,6 +469,14 @@ var _ = Describe("RBD", func() {
 				})
 			}
 
+			By("verify readAffinity support", func() {
+				err := verifyReadAffinity(f, pvcPath, appPath,
+					rbdDaemonsetName, rbdContainerName, cephCSINamespace)
+				if err != nil {
+					framework.Failf("failed to verify readAffinity: %v", err)
+				}
+			})
+
 			By("verify mountOptions support", func() {
 				err := verifySeLinuxMountOption(f, pvcPath, appPath,
 					rbdDaemonsetName, rbdContainerName, cephCSINamespace)

e2e/utils.go

@@ -827,6 +827,15 @@ func enableTopologyInTemplate(data string) string {
 	return strings.ReplaceAll(data, "--feature-gates=Topology=false", "--feature-gates=Topology=true")
 }
 
+func enableReadAffinityInTemplate(template string) string {
+	return strings.ReplaceAll(template, "# - \"--enable-read-affinity=true\"", "- \"--enable-read-affinity=true\"")
+}
+
+func addCrsuhLocationLabels(template, labels string) string {
+	return strings.ReplaceAll(template, "# - \"--crush-location-labels=topology.io/zone,topology.io/rack\"",
+		"- \"--crush-location-labels="+labels+"\"")
+}
+
 func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) (string, error) {
 	filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
 	// write data in PVC

go.mod

@@ -4,47 +4,47 @@ go 1.20
 
 require (
 	github.com/IBM/keyprotect-go-client v0.12.2
-	github.com/aws/aws-sdk-go v1.44.333
-	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5
+	github.com/aws/aws-sdk-go v1.45.24
+	github.com/aws/aws-sdk-go-v2/service/sts v1.23.1
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	// TODO: API for managing subvolume metadata and snapshot metadata requires `ceph_ci_untested` build-tag
 	github.com/ceph/go-ceph v0.23.0
 	github.com/container-storage-interface/spec v1.8.0
 	github.com/csi-addons/replication-lib-utils v0.2.0
 	github.com/csi-addons/spec v0.2.1-0.20230606140122-d20966d2e444
-	github.com/gemalto/kmip-go v0.0.9
+	github.com/gemalto/kmip-go v0.0.10
 	github.com/golang/protobuf v1.5.3
 	github.com/google/fscrypt v0.3.4
 	github.com/google/uuid v1.3.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/hashicorp/vault/api v1.9.2
+	github.com/hashicorp/vault/api v1.10.0
 	github.com/kubernetes-csi/csi-lib-utils v0.14.0
-	github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0
+	github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0
 	github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a
-	github.com/onsi/ginkgo/v2 v2.12.0
-	github.com/onsi/gomega v1.27.10
+	github.com/onsi/ginkgo/v2 v2.13.0
+	github.com/onsi/gomega v1.28.0
 	github.com/pkg/xattr v0.4.9
-	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_golang v1.17.0
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/crypto v0.12.0
-	golang.org/x/net v0.14.0
-	golang.org/x/sys v0.11.0
-	google.golang.org/grpc v1.57.0
+	golang.org/x/crypto v0.14.0
+	golang.org/x/net v0.16.0
+	golang.org/x/sys v0.13.0
+	google.golang.org/grpc v1.58.2
 	google.golang.org/protobuf v1.31.0
 	//
 	// when updating k8s.io/kubernetes, make sure to update the replace section too
 	//
-	k8s.io/api v0.28.1
-	k8s.io/apimachinery v0.28.1
+	k8s.io/api v0.28.2
+	k8s.io/apimachinery v0.28.2
 	k8s.io/client-go v12.0.0+incompatible
-	k8s.io/cloud-provider v0.28.0
+	k8s.io/cloud-provider v0.28.2
 	k8s.io/klog/v2 v2.100.1
-	k8s.io/kubernetes v1.28.1
-	k8s.io/mount-utils v0.28.0
+	k8s.io/kubernetes v1.28.2
+	k8s.io/mount-utils v0.28.2
 	k8s.io/pod-security-admission v0.0.0
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
-	sigs.k8s.io/controller-runtime v0.16.0
+	sigs.k8s.io/controller-runtime v0.16.2
 )
 
 require (
@@ -54,11 +54,11 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect
 	github.com/armon/go-metrics v0.3.10 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
-	github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
-	github.com/aws/smithy-go v1.14.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.21.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 // indirect
+	github.com/aws/smithy-go v1.15.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
@@ -68,7 +68,7 @@ require (
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.8.2+incompatible // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fatih/color v1.13.0 // indirect
@@ -87,7 +87,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/cel-go v0.16.0 // indirect
+	github.com/google/cel-go v0.16.1 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
@@ -130,9 +130,9 @@ require (
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
-	github.com/prometheus/procfs v0.10.1 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -153,27 +153,27 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.12.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
+	google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.28.0 // indirect
-	k8s.io/apiserver v0.28.0 // indirect
-	k8s.io/component-base v0.28.0 // indirect
-	k8s.io/component-helpers v0.28.0 // indirect
-	k8s.io/controller-manager v0.28.0 // indirect
-	k8s.io/kms v0.28.0 // indirect
+	k8s.io/apiserver v0.28.2 // indirect
+	k8s.io/component-base v0.28.2 // indirect
+	k8s.io/component-helpers v0.28.2 // indirect
+	k8s.io/controller-manager v0.28.2 // indirect
+	k8s.io/kms v0.28.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/kubectl v0.0.0 // indirect
 	k8s.io/kubelet v0.0.0 // indirect
@@ -193,33 +193,33 @@ replace (
 	//
 	// k8s.io/kubernetes depends on these k8s.io packages, but unversioned
 	//
-	k8s.io/api => k8s.io/api v0.28.0
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.0
-	k8s.io/apimachinery => k8s.io/apimachinery v0.28.0
-	k8s.io/apiserver => k8s.io/apiserver v0.28.0
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.0
-	k8s.io/client-go => k8s.io/client-go v0.28.0
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.0
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.0
-	k8s.io/code-generator => k8s.io/code-generator v0.28.0
-	k8s.io/component-base => k8s.io/component-base v0.28.0
-	k8s.io/component-helpers => k8s.io/component-helpers v0.28.0
-	k8s.io/controller-manager => k8s.io/controller-manager v0.28.0
-	k8s.io/cri-api => k8s.io/cri-api v0.28.0
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.0
-	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.28.0
-	k8s.io/endpointslice => k8s.io/endpointslice v0.28.0
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.0
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.0
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.0
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.0
-	k8s.io/kubectl => k8s.io/kubectl v0.28.0
-	k8s.io/kubelet => k8s.io/kubelet v0.28.0
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.0
-	k8s.io/metrics => k8s.io/metrics v0.28.0
-	k8s.io/mount-utils => k8s.io/mount-utils v0.28.0
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.0
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.0
+	k8s.io/api => k8s.io/api v0.28.2
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.2
+	k8s.io/apimachinery => k8s.io/apimachinery v0.28.2
+	k8s.io/apiserver => k8s.io/apiserver v0.28.2
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.2
+	k8s.io/client-go => k8s.io/client-go v0.28.2
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.2
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.2
+	k8s.io/code-generator => k8s.io/code-generator v0.28.2
+	k8s.io/component-base => k8s.io/component-base v0.28.2
+	k8s.io/component-helpers => k8s.io/component-helpers v0.28.2
+	k8s.io/controller-manager => k8s.io/controller-manager v0.28.2
+	k8s.io/cri-api => k8s.io/cri-api v0.28.2
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.2
+	k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.28.2
+	k8s.io/endpointslice => k8s.io/endpointslice v0.28.2
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.2
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.2
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.2
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.2
+	k8s.io/kubectl => k8s.io/kubectl v0.28.2
+	k8s.io/kubelet => k8s.io/kubelet v0.28.2
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.2
+	k8s.io/metrics => k8s.io/metrics v0.28.2
+	k8s.io/mount-utils => k8s.io/mount-utils v0.28.2
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.2
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.2
 	// layeh.com seems to be misbehaving
 	layeh.com/radius => github.com/layeh/radius v0.0.0-20190322222518-890bc1058917
 )

go.sum

@@ -37,8 +37,8 @@ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34h
 cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA=
 cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM=
 cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I=
-cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
 cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY=
+cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk=
 cloud.google.com/go/accessapproval v1.4.0/go.mod h1:zybIuC3KpDOvotz59lFe5qxRZx6C75OtwbisN56xYB4=
 cloud.google.com/go/accessapproval v1.5.0/go.mod h1:HFy3tuiGvMdcd/u+Cu5b9NkO1pEICJ46IR82PoUdplw=
 cloud.google.com/go/accessapproval v1.6.0/go.mod h1:R0EiYnwV5fsRFiKZkPHr6mwyk2wxUJ30nL4j2pcFY2E=
@@ -174,7 +174,7 @@ cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvj
 cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
 cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs=
 cloud.google.com/go/compute v1.19.0/go.mod h1:rikpw2y+UMidAe9tISo04EHNOIf42RLYF/q8Bs93scU=
-cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
+cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk=
 cloud.google.com/go/compute/metadata v0.1.0/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU=
 cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
 cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
@@ -313,8 +313,8 @@ cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQE
 cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE=
 cloud.google.com/go/iam v0.11.0/go.mod h1:9PiLDanza5D+oWFZiH1uG+RnRCfEGKoyl6yo4cgWZGY=
 cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY=
-cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
 cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
+cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y=
 cloud.google.com/go/iap v1.4.0/go.mod h1:RGFwRJdihTINIe4wZ2iCP0zF/qu18ZwyKxrhMhygBEc=
 cloud.google.com/go/iap v1.5.0/go.mod h1:UH/CGgKd4KyohZL5Pt0jSKE4m3FR51qg6FKQ/z/Ix9A=
 cloud.google.com/go/iap v1.6.0/go.mod h1:NSuvI9C/j7UdjGjIde7t7HBz+QTwBcapPE07+sSRcLk=
@@ -333,8 +333,8 @@ cloud.google.com/go/kms v1.6.0/go.mod h1:Jjy850yySiasBUDi6KFUwUv2n1+o7QZFyuUJg6O
 cloud.google.com/go/kms v1.8.0/go.mod h1:4xFEhYFqvW+4VMELtZyxomGSYtSQKzM178ylFW4jMAg=
 cloud.google.com/go/kms v1.9.0/go.mod h1:qb1tPTgfF9RQP8e1wq4cLFErVuTJv7UsSC915J8dh3w=
 cloud.google.com/go/kms v1.10.0/go.mod h1:ng3KTUtQQU9bPX3+QGLsflZIHlkbn8amFAMY63m8d24=
-cloud.google.com/go/kms v1.10.1 h1:7hm1bRqGCA1GBRQUrp831TwJ9TWhP+tvLuP497CQS2g=
 cloud.google.com/go/kms v1.10.1/go.mod h1:rIWk/TryCkR59GMC3YtHtXeLzd634lBbKenvyySAyYI=
+cloud.google.com/go/kms v1.12.1 h1:xZmZuwy2cwzsocmKDOPu4BL7umg8QXagQx6fKVmf45U=
 cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic=
 cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI=
 cloud.google.com/go/language v1.7.0/go.mod h1:DJ6dYN/W+SQOjF8e1hLQXMF21AkH2w9wiPzPCJa2MIE=
@@ -370,8 +370,8 @@ cloud.google.com/go/metastore v1.10.0/go.mod h1:fPEnH3g4JJAk+gMRnrAnoqyv2lpUCqJP
 cloud.google.com/go/monitoring v1.7.0/go.mod h1:HpYse6kkGo//7p6sT0wsIC6IBDET0RhIsnmlA53dvEk=
 cloud.google.com/go/monitoring v1.8.0/go.mod h1:E7PtoMJ1kQXWxPjB6mv2fhC5/15jInuulFdYYtlcvT4=
 cloud.google.com/go/monitoring v1.12.0/go.mod h1:yx8Jj2fZNEkL/GYZyTLS4ZtZEZN8WtDEiEqG4kLK50w=
-cloud.google.com/go/monitoring v1.13.0 h1:2qsrgXGVoRXpP7otZ14eE1I568zAa92sJSDPyOJvwjM=
 cloud.google.com/go/monitoring v1.13.0/go.mod h1:k2yMBAB1H9JT/QETjNkgdCGD9bPF712XiLTVr+cBrpw=
+cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58=
 cloud.google.com/go/networkconnectivity v1.4.0/go.mod h1:nOl7YL8odKyAOtzNX73/M5/mGZgqqMeryi6UPZTk/rA=
 cloud.google.com/go/networkconnectivity v1.5.0/go.mod h1:3GzqJx7uhtlM3kln0+x5wyFvuVH1pIBJjhCpjzSt75o=
 cloud.google.com/go/networkconnectivity v1.6.0/go.mod h1:OJOoEXW+0LAxHh89nXd64uGG+FbQoeH8DtxCHVOMlaM=
@@ -716,20 +716,20 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
-github.com/aws/aws-sdk-go v1.44.333 h1:X0j5TGXtHLZzDB/uRcGKLG77ERFtxYQtXefs+Apf2PU=
-github.com/aws/aws-sdk-go v1.44.333/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
-github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 h1:CdzPW9kKitgIiLV1+MHobfR5Xg25iYnyzWZhyQuSlDI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QGF2Rs33W5MaN9gYdEQOBBFPLwTZkEhRwI33f7KIG0o=
-github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ=
-github.com/aws/aws-sdk-go-v2/service/sts v1.21.5/go.mod h1:VC7JDqsqiwXukYEDjoHh9U0fOJtNWh04FPQz4ct4GGU=
-github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
-github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/aws-sdk-go v1.45.24 h1:TZx/CizkmCQn8Rtsb11iLYutEQVGK5PK9wAhwouELBo=
+github.com/aws/aws-sdk-go v1.45.24/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go-v2 v1.21.1 h1:wjHYshtPpYOZm+/mu3NhVgRRc0baM6LJZOmxPZ5Cwzs=
+github.com/aws/aws-sdk-go-v2 v1.21.1/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42 h1:817VqVe6wvwE46xXy6YF5RywvjOX6U2zRQQ6IbQFK0s=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.42/go.mod h1:oDfgXoBBmj+kXnqxDDnIDnC56QBosglKp8ftRCTxR+0=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36 h1:7ZApaXzWbo8slc+W5TynuUlB4z66g44h7uqa3/d/BsY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.36/go.mod h1:rwr4WnmFi3RJO0M4dxbJtgi9BPLMpVBMX1nUte5ha9U=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36 h1:YXlm7LxwNlauqb2OrinWlcvtsflTzP8GaMvYfQBhoT4=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.36/go.mod h1:ou9ffqJ9hKOVZmjlC6kQ6oROAyG1M4yBKzR+9BKbDwk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.23.1 h1:ASNYk1ypWAxRhJjKS0jBnTUeDl7HROOpeSMu1xDA/I8=
+github.com/aws/aws-sdk-go-v2/service/sts v1.23.1/go.mod h1:2cnsAhVT3mqusovc2stUSUrSBGTcX9nh8Tu6xh//2eI=
+github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8=
+github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
@@ -862,8 +862,9 @@ github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo
 github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
 github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
+github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ=
+github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -877,9 +878,10 @@ github.com/envoyproxy/go-control-plane v0.10.3/go.mod h1:fJJn/j26vwOu972OllsvAgJ
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/envoyproxy/protoc-gen-validate v0.6.7/go.mod h1:dyJXwwfPK2VSqiB9Klm1J6romD608Ba7Hij42vrOBCo=
 github.com/envoyproxy/protoc-gen-validate v0.9.1/go.mod h1:OKNgG7TCp5pF4d6XftA0++PMirau2/yoOwVac3AbF2w=
-github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8=
+github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
 github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
@@ -906,8 +908,8 @@ github.com/gammazero/deque v0.0.0-20190130191400-2afb3858e9c7/go.mod h1:GeIq9qoE
 github.com/gammazero/workerpool v0.0.0-20190406235159-88d534f22b56/go.mod h1:w9RqFVO2BM3xwWEcAB8Fwp0OviTBBEiRmSBDfbXnd3w=
 github.com/gemalto/flume v0.13.0 h1:EEeQvAxyFys3BH8IxEU7ZpM6Kr1sYn20HuZq6dgyMR8=
 github.com/gemalto/flume v0.13.0/go.mod h1:3iOEZiK/HD8SnFTqHCQoOHQKaHlBY0b6z55P8SLaOzk=
-github.com/gemalto/kmip-go v0.0.9 h1:PMm0j3k5l7H6BIp13+V6aq0mvp0O3YlKVHkDweLkzik=
-github.com/gemalto/kmip-go v0.0.9/go.mod h1:YBUgKtkYpGCL+xA1oClQrMYTWDvHN/A2wSh2LmT5+JY=
+github.com/gemalto/kmip-go v0.0.10 h1:jAAZejUdRrspKigLoA62MTmIj0T7DDDOzdxHi1cDjoU=
+github.com/gemalto/kmip-go v0.0.10/go.mod h1:7XtwjeX7tNQt/FoDZDWXjYOkyV26ZQF1fKFBeR3mCwY=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew=
 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I=
@@ -1040,8 +1042,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/cel-go v0.16.0 h1:DG9YQ8nFCFXAs/FDDwBxmL1tpKNrdlGUM9U3537bX/Y=
-github.com/google/cel-go v0.16.0/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
+github.com/google/cel-go v0.16.1 h1:3hZfSNiAU3KOiNtxuFXVp5WFy4hf/Ly3Sa4/7F8SXNo=
+github.com/google/cel-go v0.16.1/go.mod h1:HXZKzB0LXqer5lHHgfWAnlYwJaQBDKMjxjulNQzhwhY=
 github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/fscrypt v0.3.4 h1:XGSVMIsQFooj82aRRfYn3JpgU/4fOTnzXPnjhxC8uH8=
 github.com/google/fscrypt v0.3.4/go.mod h1:BRpw7vaeDitXGRvXa281i/ivQszAdBIiUYDWHjVTkcs=
@@ -1292,8 +1294,8 @@ github.com/hashicorp/vault/api v1.0.5-0.20191122173911-80fcc7907c78/go.mod h1:Uf
 github.com/hashicorp/vault/api v1.0.5-0.20200215224050-f6547fa8e820/go.mod h1:3f12BMfgDGjTsTtIUj+ZKZwSobQpZtYGFIEehOv5z1o=
 github.com/hashicorp/vault/api v1.0.5-0.20200317185738-82f498082f02/go.mod h1:3f12BMfgDGjTsTtIUj+ZKZwSobQpZtYGFIEehOv5z1o=
 github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a/go.mod h1:R3Umvhlxi2TN7Ex2hzOowyeNb+SfbVWI973N+ctaFMk=
-github.com/hashicorp/vault/api v1.9.2 h1:YjkZLJ7K3inKgMZ0wzCU9OHqc+UqMQyXsPXnf3Cl2as=
-github.com/hashicorp/vault/api v1.9.2/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8=
+github.com/hashicorp/vault/api v1.10.0 h1:/US7sIjWN6Imp4o/Rj1Ce2Nr5bki/AXi9vAW3p2tOJQ=
+github.com/hashicorp/vault/api v1.10.0/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8=
 github.com/hashicorp/vault/sdk v0.1.8/go.mod h1:tHZfc6St71twLizWNHvnnbiGFo1aq0eD2jGPLtP8kAU=
 github.com/hashicorp/vault/sdk v0.1.14-0.20190730042320-0dc007d98cc8/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
 github.com/hashicorp/vault/sdk v0.1.14-0.20191108161836-82f2b5571044/go.mod h1:PcekaFGiPJyHnFy+NZhP6ll650zEw51Ag7g/YEa+EOU=
@@ -1407,8 +1409,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubernetes-csi/csi-lib-utils v0.14.0 h1:pusB32LkSd7GhuT8Z6cyRFqByujc28ygWV97ndaT19s=
 github.com/kubernetes-csi/csi-lib-utils v0.14.0/go.mod h1:uX8xidqxGJOLXtsfCCVsxWtZl/9NiLyd2DD3Nb+KoP4=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
-github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 h1:cMM5AB37e9aRGjErygVT6EuBPB6s5a+l95OPERmSlVM=
-github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0/go.mod h1:VQVLCPGDX5l6V5PezjlDXLa+SpCbWSVU7B16cFWVVeE=
+github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0 h1:qS4r4ljINLWKJ9m9Ge3Q3sGZ/eIoDVDT2RhAdQFHb1k=
+github.com/kubernetes-csi/external-snapshotter/client/v6 v6.3.0/go.mod h1:oGXx2XTEzs9ikW2V6IC1dD8trgjRsS/Mvc2JRiC618Y=
 github.com/layeh/radius v0.0.0-20190322222518-890bc1058917/go.mod h1:fywZKyu//X7iRzaxLgPWsvc0L26IUpVvE/aeIL2JtIQ=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/libopenstorage/autopilot-api v0.6.1-0.20210128210103-5fbb67948648/go.mod h1:6JLrPbR3ZJQFbUY/+QJMl/aF00YdIrLf8/GWAplgvJs=
@@ -1534,8 +1536,8 @@ github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxm
 github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=
 github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts=
 github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLywzIhbKM=
-github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI=
-github.com/onsi/ginkgo/v2 v2.12.0/go.mod h1:ZNEzXISYlqpb8S36iN71ifqLi3vVD1rVJGvWRCJOUpQ=
+github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
+github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -1553,8 +1555,8 @@ github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfad
 github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw=
 github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
 github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
-github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI=
-github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M=
+github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c=
+github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@@ -1638,16 +1640,18 @@ github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
-github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
 github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
-github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
 github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
@@ -1674,8 +1678,9 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
-github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rboyer/safeio v0.2.1 h1:05xhhdRNAdS3apYm7JRjOqngf4xruaW959jmRxGDuSU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -1909,8 +1914,8 @@ golang.org/x/crypto v0.0.0-20220408190544-5352b0902921/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2053,8 +2058,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
+golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -2087,8 +2092,9 @@ golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri
 golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw=
-golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
+golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
+golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -2225,8 +2231,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -2239,8 +2245,8 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
 golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
-golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
-golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2259,8 +2265,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -2583,14 +2589,17 @@ google.golang.org/genproto v0.0.0-20230323212658-478b75c54725/go.mod h1:UUQDJDOl
 google.golang.org/genproto v0.0.0-20230330154414-c0448cd141ea/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633/go.mod h1:UUQDJDOlWu4KYeJZffbWgBkS1YFobzKbLVfK69pe0Ak=
 google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
-google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 h1:9NWlQfY2ePejTmfwUH1OWwmznFa+0kKcHGPDvcPza9M=
 google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk=
+google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g=
+google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0=
 google.golang.org/genproto/googleapis/api v0.0.0-20230525234020-1aefcd67740a/go.mod h1:ts19tUU+Z0ZShN1y3aPyq2+O3d5FUNNgT6FtOzmrNn8=
-google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 h1:m8v1xLLLzMe1m5P+gCTF8nJB9epwZQUBERm20Oy1poQ=
 google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw=
+google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234015-3fc162c6f38a/go.mod h1:xURIpW9ES5+/GZhnV6beoEtxQrnkRGIfP5VQG2tCBLc=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
@@ -2637,8 +2646,8 @@ google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD
 google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
 google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw=
 google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g=
-google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
-google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
+google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I=
+google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -2722,26 +2731,26 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 honnef.co/go/tools v0.3.0/go.mod h1:vlRD9XErLMGT+mDuofSr0mMMquscM/1nQqtRSsh6m70=
-k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM=
-k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY=
-k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E=
-k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE=
-k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA=
-k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw=
-k8s.io/apiserver v0.28.0 h1:wVh7bK6Xj7hq+5ntInysTeQRAOqqFoKGUOW2yj8DXrY=
-k8s.io/apiserver v0.28.0/go.mod h1:MvLmtxhQ0Tb1SZk4hfJBjs8iqr5nhYeaFSaoEcz7Lk4=
-k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM=
-k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc=
-k8s.io/cloud-provider v0.28.0 h1:BTIW7b757T+VXB5yqJeajPXsNOmeooopUgfzQueiWvk=
-k8s.io/cloud-provider v0.28.0/go.mod h1:u0MGqdlutkTmCJyNrCzIMJ+OhrwQE9x5X8mBTN0R7us=
-k8s.io/code-generator v0.28.0/go.mod h1:ueeSJZJ61NHBa0ccWLey6mwawum25vX61nRZ6WOzN9A=
-k8s.io/component-base v0.28.0 h1:HQKy1enJrOeJlTlN4a6dU09wtmXaUvThC0irImfqyxI=
-k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk=
-k8s.io/component-helpers v0.28.0 h1:ubHUiEF7H/DOx4471pHHsLlH3EGu8jlEvnld5PS4KdI=
-k8s.io/component-helpers v0.28.0/go.mod h1:i7hJ/oFhZImqUWwjLFG/yGkLpJ3KFoirY2DLYIMql6Q=
-k8s.io/controller-manager v0.28.0 h1:55rmyzwEOnhAZLsuDdDHwVT2sGzkleFY0SqZFKsLN5U=
-k8s.io/controller-manager v0.28.0/go.mod h1:WrABGmrpEWBap27eu533RpW5lBnVT5K+u2oc2bDwcmU=
-k8s.io/csi-translation-lib v0.28.0 h1:X3Kr5aHvH4xutNg4pgdc6RP0h3FOlJGDeui5CLfBeO4=
+k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw=
+k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg=
+k8s.io/apiextensions-apiserver v0.28.2 h1:J6/QRWIKV2/HwBhHRVITMLYoypCoPY1ftigDM0Kn+QU=
+k8s.io/apiextensions-apiserver v0.28.2/go.mod h1:5tnkxLGa9nefefYzWuAlWZ7RZYuN/765Au8cWLA6SRg=
+k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ=
+k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU=
+k8s.io/apiserver v0.28.2 h1:rBeYkLvF94Nku9XfXyUIirsVzCzJBs6jMn3NWeHieyI=
+k8s.io/apiserver v0.28.2/go.mod h1:f7D5e8wH8MWcKD7azq6Csw9UN+CjdtXIVQUyUhrtb+E=
+k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY=
+k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY=
+k8s.io/cloud-provider v0.28.2 h1:9qsYm86hm4bnPgZbl9LE29Zfgjuq3NZR2dgtPioJ40s=
+k8s.io/cloud-provider v0.28.2/go.mod h1:40fqf6MtgYho5Eu4gkyLgh5abxU/QKTMTIwBxt4ILyU=
+k8s.io/code-generator v0.28.2/go.mod h1:ueeSJZJ61NHBa0ccWLey6mwawum25vX61nRZ6WOzN9A=
+k8s.io/component-base v0.28.2 h1:Yc1yU+6AQSlpJZyvehm/NkJBII72rzlEsd6MkBQ+G0E=
+k8s.io/component-base v0.28.2/go.mod h1:4IuQPQviQCg3du4si8GpMrhAIegxpsgPngPRR/zWpzc=
+k8s.io/component-helpers v0.28.2 h1:r/XJ265PMirW9EcGXr/F+2yWrLPo2I69KdvcY/h9HAo=
+k8s.io/component-helpers v0.28.2/go.mod h1:pF1R5YWQ+sgf0i6EbVm+MQCzkYuqutDUibdrkvAa6aI=
+k8s.io/controller-manager v0.28.2 h1:C2RKx+NH3Iw+4yLdTGNJlYUd4cRV1N8tKl4XfqMwuTk=
+k8s.io/controller-manager v0.28.2/go.mod h1:7bT6FlTE96Co7QevCtvcVnZZIJSaGj6F7EmyT2Rf3GY=
+k8s.io/csi-translation-lib v0.28.2 h1:63MIOXUn5bet2Mw7G+A7zFmLzQ/vzBrjvNYIlXYh/n0=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20220902162205-c0856e24416d/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
@@ -2751,21 +2760,21 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=
 k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kms v0.28.0 h1:BwJhU9qPcJhHLUcQjtelOSjYti+1/caJLr+4jHbKzTA=
-k8s.io/kms v0.28.0/go.mod h1:CNU792ls92v2Ye7Vn1jn+xLqYtUSezDZNVu6PLbJyrU=
+k8s.io/kms v0.28.2 h1:KhG63LHopCdzs1oKA1j+NWleuIXudgOyCqJo4yi3GaM=
+k8s.io/kms v0.28.2/go.mod h1:iAjgIqBrV2+8kmsjbbgUkAyKSuYq5g1dW9knpt6OhaE=
 k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ=
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
-k8s.io/kubectl v0.28.0 h1:qhfju0OaU+JGeBlToPeeIg2UJUWP++QwTkpio6nlPKg=
-k8s.io/kubectl v0.28.0/go.mod h1:1We+E5nSX3/TVoSQ6y5Bzld5OhTBHZHlKEYl7g/NaTk=
-k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U=
-k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc=
-k8s.io/kubernetes v1.28.1 h1:ZQuukGbpVjSbMypkjNErpbsSHni6RPgoqz+2zDBsuMY=
-k8s.io/kubernetes v1.28.1/go.mod h1:rBQpjGYlLBV0KuOLw8EG45N5EBCskWiPpi0xy5liHMI=
-k8s.io/mount-utils v0.28.0 h1:BGYxriZPWTJFCEWDtXsdC1ZPFvI6HbfXCWpjJ42mIw4=
-k8s.io/mount-utils v0.28.0/go.mod h1:AyP8LmZSLgpGdFQr+vzHTerlPiGvXUdP99n98Er47jw=
-k8s.io/pod-security-admission v0.28.0 h1:Vz8XTjMAKHQFZv9Q4GdmO59CUtelkPPDRJTy/WTTc3g=
-k8s.io/pod-security-admission v0.28.0/go.mod h1:hABVUcP7SRALDvESOK+RYIAWc9uZ5I1eSdcUwsOYTU8=
+k8s.io/kubectl v0.28.2 h1:fOWOtU6S0smdNjG1PB9WFbqEIMlkzU5ahyHkc7ESHgM=
+k8s.io/kubectl v0.28.2/go.mod h1:6EQWTPySF1fn7yKoQZHYf9TPwIl2AygHEcJoxFekr64=
+k8s.io/kubelet v0.28.2 h1:wqe5zKtVhNWwtdABU0mpcWVe8hc6VdVvs2kqQridZRw=
+k8s.io/kubelet v0.28.2/go.mod h1:rvd0e7T5TjPcfZvy62P90XhFzp0IhPIOy+Pqy3Rtipo=
+k8s.io/kubernetes v1.28.2 h1:GhcnYeNTukeaC0dD5BC+UWBvzQsFEpWj7XBVMQptfYc=
+k8s.io/kubernetes v1.28.2/go.mod h1:FmB1Mlp9ua0ezuwQCTGs/y6wj/fVisN2sVxhzjj0WDk=
+k8s.io/mount-utils v0.28.2 h1:sIdMH7fRhcU48V1oYJ9cLmLm/TG+2jLhhe8eS3I+FWg=
+k8s.io/mount-utils v0.28.2/go.mod h1:AyP8LmZSLgpGdFQr+vzHTerlPiGvXUdP99n98Er47jw=
+k8s.io/pod-security-admission v0.28.2 h1:3kiOL+gc6auNTGHuQ0hVsGxYu2YO/7DZb0xYR84GxiQ=
+k8s.io/pod-security-admission v0.28.2/go.mod h1:gReea39xbhIzf4Ry0FDuiTi8uj1N5R9YXOh8zQSuTxs=
 k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
@@ -2811,8 +2820,8 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6UvFu4WzQDrN7Pga4a7Qg+zcfcj64PA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0=
 sigs.k8s.io/controller-runtime v0.2.2/go.mod h1:9dyohw3ZtoXQuV1e766PHUn+cmrRCIcBh6XIMFNMZ+I=
-sigs.k8s.io/controller-runtime v0.16.0 h1:5koYaaRVBHDr0LZAJjO5dWzUjMsh6cwa7q1Mmusrdvk=
-sigs.k8s.io/controller-runtime v0.16.0/go.mod h1:77DnuwA8+J7AO0njzv3wbNlMOnGuLrwFr8JPNwx3J7g=
+sigs.k8s.io/controller-runtime v0.16.2 h1:mwXAVuEk3EQf478PQwQ48zGOXvW27UJc8NHktQVuIPU=
+sigs.k8s.io/controller-runtime v0.16.2/go.mod h1:vpMu3LpI5sYWtujJOa2uPK61nB5rbwlN7BAB8aSLvGU=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=

internal/cephfs/core/metadata.go

@@ -51,6 +51,32 @@ func (s *subVolumeClient) isUnsupportedSubVolMetadata(err error) bool {
 	return true
 }
 
+// isSubVolumeGroupCreated returns true if subvolume group is created.
+func (s *subVolumeClient) isSubVolumeGroupCreated() bool {
+	newLocalClusterState(s.clusterID)
+	clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RLock()
+	defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RUnlock()
+
+	if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil {
+		return false
+	}
+
+	return clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup]
+}
+
+// updateSubVolumeGroupCreated updates subvolume group created map.
+// If the map is nil, it creates a new map and updates it.
+func (s *subVolumeClient) updateSubVolumeGroupCreated(state bool) {
+	clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Lock()
+	defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Unlock()
+
+	if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil {
+		clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated = make(map[string]bool)
+	}
+
+	clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup] = state
+}
+
 // setMetadata sets custom metadata on the subvolume in a volume as a
 // key-value pair.
 func (s *subVolumeClient) setMetadata(key, value string) error {

internal/cephfs/core/volume.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"path"
 	"strings"
+	"sync"
 
 	cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors"
 	fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
@@ -32,12 +33,17 @@ import (
 	"github.com/ceph/go-ceph/rados"
 )
 
-// clusterAdditionalInfo contains information regarding if resize is
-// supported in the particular cluster and subvolumegroup is
-// created or not.
-// Subvolumegroup creation and volume resize decisions are
-// taken through this additional cluster information.
-var clusterAdditionalInfo = make(map[string]*localClusterState)
+var (
+	// clusterAdditionalInfo contains information regarding if resize is
+	// supported in the particular cluster and subvolumegroup is
+	// created or not.
+	// Subvolumegroup creation and volume resize decisions are
+	// taken through this additional cluster information.
+	clusterAdditionalInfo = make(map[string]*localClusterState)
+	// clusterAdditionalInfoMutex is used to protect against
+	// concurrent writes.
+	clusterAdditionalInfoMutex = sync.Mutex{}
+)
 
 // Subvolume holds subvolume information. This includes only the needed members
 // from fsAdmin.SubVolumeInfo.
@@ -209,14 +215,18 @@ type localClusterState struct {
 	// set true once a subvolumegroup is created
 	// for corresponding filesystem in a cluster.
 	subVolumeGroupsCreated map[string]bool
+	// subVolumeGroupsRWMutex is used to protect subVolumeGroupsCreated map
+	//  against concurrent writes while allowing multiple readers.
+	subVolumeGroupsRWMutex sync.RWMutex
 }
 
 func newLocalClusterState(clusterID string) {
 	// verify if corresponding clusterID key is present in the map,
 	// and if not, initialize with default values(false).
+	clusterAdditionalInfoMutex.Lock()
+	defer clusterAdditionalInfoMutex.Unlock()
 	if _, keyPresent := clusterAdditionalInfo[clusterID]; !keyPresent {
 		clusterAdditionalInfo[clusterID] = &localClusterState{}
-		clusterAdditionalInfo[clusterID].subVolumeGroupsCreated = make(map[string]bool)
 	}
 }
 
@@ -232,7 +242,7 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error {
 	}
 
 	// create subvolumegroup if not already created for the cluster.
-	if !clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] {
+	if !s.isSubVolumeGroupCreated() {
 		opts := fsAdmin.SubVolumeGroupOptions{}
 		err = ca.CreateSubVolumeGroup(s.FsName, s.SubvolumeGroup, &opts)
 		if err != nil {
@@ -246,7 +256,7 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error {
 			return err
 		}
 		log.DebugLog(ctx, "cephfs: created subvolume group %s", s.SubvolumeGroup)
-		clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] = true
+		s.updateSubVolumeGroupCreated(true)
 	}
 
 	opts := fsAdmin.SubVolumeOptions{
@@ -264,7 +274,7 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error {
 		if errors.Is(err, rados.ErrNotFound) {
 			// Reset the subVolumeGroupsCreated so that we can try again to create the
 			// subvolumegroup in next request if the error is Not Found.
-			clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.FsName] = false
+			s.updateSubVolumeGroupCreated(false)
 		}
 
 		return err

internal/csi-addons/rbd/replication.go

@@ -243,7 +243,11 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context,
 	defer rs.VolumeLocks.Release(volumeID)
 
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):
@@ -305,7 +309,11 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context,
 	defer rs.VolumeLocks.Release(volumeID)
 
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):
@@ -376,7 +384,11 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context,
 	defer rs.VolumeLocks.Release(volumeID)
 
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):
@@ -472,7 +484,11 @@ func (rs *ReplicationServer) DemoteVolume(ctx context.Context,
 	defer rs.VolumeLocks.Release(volumeID)
 
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):
@@ -585,7 +601,11 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context,
 	}
 	defer rs.VolumeLocks.Release(volumeID)
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):
@@ -798,7 +818,11 @@ func (rs *ReplicationServer) GetVolumeReplicationInfo(ctx context.Context,
 	}
 	defer rs.VolumeLocks.Release(volumeID)
 	rbdVol, err := corerbd.GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, corerbd.ErrImageNotFound):

internal/rbd/controllerserver.go

@@ -922,7 +922,11 @@ func (cs *ControllerServer) DeleteVolume(
 	}
 
 	rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		return cs.checkErrAndUndoReserve(ctx, err, volumeID, rbdVol, cr)
 	}
@@ -1078,7 +1082,11 @@ func (cs *ControllerServer) CreateSnapshot(
 
 	// Fetch source volume information
 	rbdVol, err := GenVolFromVolID(ctx, req.GetSourceVolumeId(), cr, req.GetSecrets())
-	defer rbdVol.Destroy()
+	defer func() {
+		if rbdVol != nil {
+			rbdVol.Destroy()
+		}
+	}()
 	if err != nil {
 		switch {
 		case errors.Is(err, ErrImageNotFound):
@@ -1334,8 +1342,6 @@ func (cs *ControllerServer) doSnapshotClone(
 
 	err = cloneRbd.createSnapshot(ctx, rbdSnap)
 	if err != nil {
-		// update rbd image name for logging
-		rbdSnap.RbdImageName = cloneRbd.RbdImageName
 		log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err)
 
 		return cloneRbd, err

internal/rbd/rbd_util.go

@@ -1411,6 +1411,7 @@ func (ri *rbdImage) hasSnapshotFeature() bool {
 }
 
 func (ri *rbdImage) createSnapshot(ctx context.Context, pOpts *rbdSnapshot) error {
+	pOpts.RbdImageName = ri.RbdImageName
 	log.DebugLog(ctx, "rbd: snap create %s using mon %s", pOpts, pOpts.Monitors)
 	image, err := ri.open()
 	if err != nil {

scripts/install-helm.sh

@@ -24,6 +24,10 @@ NODE_LABEL_REGION="test.failure-domain/region"
 NODE_LABEL_ZONE="test.failure-domain/zone"
 REGION_VALUE="testregion"
 ZONE_VALUE="testzone"
+CRUSH_LOCATION_REGION_LABEL="topology.kubernetes.io/region"
+CRUSH_LOCATION_ZONE_LABEL="topology.kubernetes.io/zone"
+CRUSH_LOCATION_REGION_VALUE="east"
+CRUSH_LOCATION_ZONE_VALUE="east-zone1"
 
 example() {
     echo "examples:" >&2
@@ -154,6 +158,8 @@ install_cephcsi_helm_charts() {
     for node in $(kubectl_retry get node -o jsonpath='{.items[*].metadata.name}'); do
         kubectl_retry label node/"${node}" ${NODE_LABEL_REGION}=${REGION_VALUE}
         kubectl_retry label node/"${node}" ${NODE_LABEL_ZONE}=${ZONE_VALUE}
+        kubectl_retry label node/"${node}" ${CRUSH_LOCATION_REGION_LABEL}=${CRUSH_LOCATION_REGION_VALUE}
+        kubectl_retry label node/"${node}" ${CRUSH_LOCATION_ZONE_LABEL}=${CRUSH_LOCATION_ZONE_VALUE}
     done
 
     # deploy storageclass if DEPLOY_SC flag is set
@@ -179,7 +185,7 @@ install_cephcsi_helm_charts() {
     kubectl_retry delete cm ceph-config --namespace "${NAMESPACE}"
 
     # shellcheck disable=SC2086
-    "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.replicaCount=1 --set-json='commonLabels={"app.kubernetes.io/name": "ceph-csi-rbd", "app.kubernetes.io/managed-by": "helm"}' ${SET_SC_TEMPLATE_VALUES} ${RBD_SECRET_TEMPLATE_VALUES} ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 --set provisioner.minSnapshotsOnImage=2
+    "${HELM}" install --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.replicaCount=1 --set-json='commonLabels={"app.kubernetes.io/name": "ceph-csi-rbd", "app.kubernetes.io/managed-by": "helm"}' ${SET_SC_TEMPLATE_VALUES} ${RBD_SECRET_TEMPLATE_VALUES} ${RBD_CHART_NAME} "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" --set provisioner.maxSnapshotsOnImage=3 --set provisioner.minSnapshotsOnImage=2 --set readAffinity.enabled=true --set readAffinity.crushLocationLabels="{${CRUSH_LOCATION_REGION_LABEL},${CRUSH_LOCATION_ZONE_LABEL}}"
 
     check_deployment_status app=ceph-csi-rbd "${NAMESPACE}"
     check_daemonset_status app=ceph-csi-rbd "${NAMESPACE}"
@@ -191,6 +197,8 @@ cleanup_cephcsi_helm_charts() {
     for node in $(kubectl_retry get node --no-headers | cut -f 1 -d ' '); do
         kubectl_retry label node/"$node" test.failure-domain/region-
         kubectl_retry label node/"$node" test.failure-domain/zone-
+        kubectl_retry label node/"$node" "${CRUSH_LOCATION_REGION_LABEL}"-
+        kubectl_retry label node/"$node" "${CRUSH_LOCATION_ZONE_LABEL}"-
     done
     # TODO/LATER we could remove the CSI labels that would have been set as well
     NAMESPACE=$1

scripts/minikube.sh

@@ -255,8 +255,6 @@ up)
         install_podman_wrapper
     fi
 
-    disable_storage_addons
-
     #  get kubernetes version we are operating on and accordingly enable feature gates
     KUBE_MAJOR=$(kube_version 1)
     KUBE_MINOR=$(kube_version 2)
@@ -283,6 +281,7 @@ up)
     if [[ "${VM_DRIVER}" = "podman" ]]; then
         ${minikube} ssh "sudo mount -oremount,rw /sys"
     fi
+    disable_storage_addons
     ${minikube} kubectl -- cluster-info
     ;;
 down)

scripts/rook.sh

@@ -1,9 +1,8 @@
 #!/bin/bash -E
 
-ROOK_VERSION=${ROOK_VERSION:-"v1.6.2"}
+ROOK_VERSION=${ROOK_VERSION:-"v1.12.5"}
 ROOK_DEPLOY_TIMEOUT=${ROOK_DEPLOY_TIMEOUT:-300}
-ROOK_URL="https://raw.githubusercontent.com/rook/rook/${ROOK_VERSION}/"
-ROOK_DEPLOYMENT_PATH="cluster/examples/kubernetes/ceph"
+ROOK_URL="https://raw.githubusercontent.com/rook/rook/${ROOK_VERSION}/deploy/examples"
 ROOK_BLOCK_POOL_NAME=${ROOK_BLOCK_POOL_NAME:-"newrbdpool"}
 ROOK_BLOCK_EC_POOL_NAME=${ROOK_BLOCK_EC_POOL_NAME:-"ec-pool"}
 
@@ -30,31 +29,10 @@ function log_errors() {
 	exit 1
 }
 
-rook_version() {
-	echo "${ROOK_VERSION#v}" | cut -d'.' -f"${1}"
-}
-
-function update_rook_url() {
-	ROOK_MAJOR=$(rook_version 1)
-	ROOK_MINOR=$(rook_version 2)
-
-	# If rook version is => 1.8 update deployment path.
-	if [ "${ROOK_MAJOR}" -eq 1 ] && [ "${ROOK_MINOR}" -ge 8 ]; then
-		ROOK_DEPLOYMENT_PATH="deploy/examples"
-	fi
-	ROOK_URL+=${ROOK_DEPLOYMENT_PATH}
-}
-
 function deploy_rook() {
 	kubectl_retry create -f "${ROOK_URL}/common.yaml"
+	kubectl_retry create -f "${ROOK_URL}/crds.yaml"
 
-	ROOK_MAJOR=$(rook_version 1)
-	ROOK_MINOR=$(rook_version 2)
-
-	# If rook version is > 1.5 , we will apply CRDs.
-	if [ "${ROOK_MAJOR}" -eq 1 ] && [ "${ROOK_MINOR}" -ge 5 ]; then
-		kubectl_retry create -f "${ROOK_URL}/crds.yaml"
-	fi
 	TEMP_DIR="$(mktemp -d)"
 	curl -o "${TEMP_DIR}/operator.yaml" "${ROOK_URL}/operator.yaml"
 	# disable rook deployed csi drivers
@@ -106,12 +84,8 @@ function teardown_rook() {
 	kubectl delete -f "${ROOK_URL}/toolbox.yaml"
 	kubectl delete -f "${ROOK_URL}/cluster-test.yaml"
 	kubectl delete -f "${ROOK_URL}/operator.yaml"
-	ROOK_MAJOR=$(rook_version 1)
-	ROOK_MINOR=$(rook_version 2)
-	if [ "${ROOK_MAJOR}" -eq 1 ] && [ "${ROOK_MINOR}" -ge 5 ]; then
-		kubectl delete -f "${ROOK_URL}/crds.yaml"
-	fi
 	kubectl delete -f "${ROOK_URL}/common.yaml"
+	kubectl delete -f "${ROOK_URL}/crds.yaml"
 }
 
 function create_block_pool() {
@@ -252,9 +226,6 @@ function check_rbd_stat() {
 	echo ""
 }
 
-# update rook URL before doing any operation.
-update_rook_url
-
 case "${1:-}" in
 deploy)
 	deploy_rook

vendor/github.com/aws/aws-sdk-go-v2/aws/go_module_metadata.go

@@ -3,4 +3,4 @@
 package aws
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.21.0"
+const goModuleVersion = "1.21.1"

vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.1.42 (2023-10-06)
+
+* **Dependency Update**: Updated to the latest SDK module versions
+
 # v1.1.41 (2023-08-21)
 
 * **Dependency Update**: Updated to the latest SDK module versions

vendor/github.com/aws/aws-sdk-go-v2/internal/configsources/go_module_metadata.go

@@ -3,4 +3,4 @@
 package configsources
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.1.41"
+const goModuleVersion = "1.1.42"

vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn/partitions.json

@@ -4,6 +4,7 @@
     "outputs" : {
       "dnsSuffix" : "amazonaws.com",
       "dualStackDnsSuffix" : "api.aws",
+      "implicitGlobalRegion" : "us-east-1",
       "name" : "aws",
       "supportsDualStack" : true,
       "supportsFIPS" : true
@@ -103,6 +104,7 @@
     "outputs" : {
       "dnsSuffix" : "amazonaws.com.cn",
       "dualStackDnsSuffix" : "api.amazonwebservices.com.cn",
+      "implicitGlobalRegion" : "cn-northwest-1",
       "name" : "aws-cn",
       "supportsDualStack" : true,
       "supportsFIPS" : true
@@ -124,6 +126,7 @@
     "outputs" : {
       "dnsSuffix" : "amazonaws.com",
       "dualStackDnsSuffix" : "api.aws",
+      "implicitGlobalRegion" : "us-gov-west-1",
       "name" : "aws-us-gov",
       "supportsDualStack" : true,
       "supportsFIPS" : true
@@ -145,6 +148,7 @@
     "outputs" : {
       "dnsSuffix" : "c2s.ic.gov",
       "dualStackDnsSuffix" : "c2s.ic.gov",
+      "implicitGlobalRegion" : "us-iso-east-1",
       "name" : "aws-iso",
       "supportsDualStack" : false,
       "supportsFIPS" : true
@@ -166,6 +170,7 @@
     "outputs" : {
       "dnsSuffix" : "sc2s.sgov.gov",
       "dualStackDnsSuffix" : "sc2s.sgov.gov",
+      "implicitGlobalRegion" : "us-isob-east-1",
       "name" : "aws-iso-b",
       "supportsDualStack" : false,
       "supportsFIPS" : true
@@ -184,6 +189,7 @@
     "outputs" : {
       "dnsSuffix" : "cloud.adc-e.uk",
       "dualStackDnsSuffix" : "cloud.adc-e.uk",
+      "implicitGlobalRegion" : "eu-isoe-west-1",
       "name" : "aws-iso-e",
       "supportsDualStack" : false,
       "supportsFIPS" : true
@@ -195,6 +201,7 @@
     "outputs" : {
       "dnsSuffix" : "csp.hci.ic.gov",
       "dualStackDnsSuffix" : "csp.hci.ic.gov",
+      "implicitGlobalRegion" : "us-isof-south-1",
       "name" : "aws-iso-f",
       "supportsDualStack" : false,
       "supportsFIPS" : true

vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v2.4.36 (2023-10-06)
+
+* **Dependency Update**: Updated to the latest SDK module versions
+
 # v2.4.35 (2023-08-21)
 
 * **Dependency Update**: Updated to the latest SDK module versions

vendor/github.com/aws/aws-sdk-go-v2/internal/endpoints/v2/go_module_metadata.go

@@ -3,4 +3,4 @@
 package endpoints
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "2.4.35"
+const goModuleVersion = "2.4.36"

vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.9.36 (2023-10-06)
+
+* **Dependency Update**: Updated to the latest SDK module versions
+
 # v1.9.35 (2023-08-21)
 
 * **Dependency Update**: Updated to the latest SDK module versions

vendor/github.com/aws/aws-sdk-go-v2/service/internal/presigned-url/go_module_metadata.go

@@ -3,4 +3,4 @@
 package presignedurl
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.9.35"
+const goModuleVersion = "1.9.36"

vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md

@@ -1,3 +1,16 @@
+# v1.23.1 (2023-10-06)
+
+* **Dependency Update**: Updated to the latest SDK module versions
+
+# v1.23.0 (2023-10-02)
+
+* **Feature**: STS API updates for assumeRole
+
+# v1.22.0 (2023-09-18)
+
+* **Announcement**: [BREAKFIX] Change in MaxResults datatype from value to pointer type in cognito-sync service.
+* **Feature**: Adds several endpoint ruleset changes across all models: smaller rulesets, removed non-unique regional endpoints, fixes FIPS and DualStack endpoints, and make region not required in SDK::Endpoint. Additional breakfix to cognito-sync field.
+
 # v1.21.5 (2023-08-21)
 
 * **Dependency Update**: Updated to the latest SDK module versions

vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go

@@ -3,4 +3,4 @@
 package sts
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.21.5"
+const goModuleVersion = "1.23.1"

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/awsinternal.go

@@ -0,0 +1,4 @@
+// DO NOT EDIT
+package corehandlers
+
+const isAwsInternal = ""

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/user_agent.go

@@ -35,3 +35,13 @@ var AddHostExecEnvUserAgentHander = request.NamedHandler{
 		request.AddToUserAgent(r, execEnvUAKey+"/"+v)
 	},
 }
+
+var AddAwsInternal = request.NamedHandler{
+	Name: "core.AddAwsInternal",
+	Fn: func(r *request.Request) {
+		if len(isAwsInternal) == 0 {
+			return
+		}
+		request.AddToUserAgent(r, isAwsInternal)
+	},
+}

vendor/github.com/aws/aws-sdk-go/aws/defaults/defaults.go

@@ -74,6 +74,7 @@ func Handlers() request.Handlers {
 	handlers.Validate.PushBackNamed(corehandlers.ValidateEndpointHandler)
 	handlers.Validate.AfterEachFn = request.HandlerListStopOnError
 	handlers.Build.PushBackNamed(corehandlers.SDKVersionUserAgentHandler)
+	handlers.Build.PushBackNamed(corehandlers.AddAwsInternal)
 	handlers.Build.PushBackNamed(corehandlers.AddHostExecEnvUserAgentHander)
 	handlers.Build.AfterEachFn = request.HandlerListStopOnError
 	handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)

vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go

@@ -389,8 +389,15 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
 		updateString(&cfg.Region, section, regionKey)
 		updateString(&cfg.CustomCABundle, section, customCABundleKey)
 
+		// we're retaining a behavioral quirk with this field that existed before
+		// the removal of literal parsing for (aws-sdk-go-v2/#2276):
+		//   - if the key is missing, the config field will not be set
+		//   - if the key is set to a non-numeric, the config field will be set to 0
 		if section.Has(roleDurationSecondsKey) {
-			d := time.Duration(section.Int(roleDurationSecondsKey)) * time.Second
+			var d time.Duration
+			if v, ok := section.Int(roleDurationSecondsKey); ok {
+				d = time.Duration(v) * time.Second
+			}
 			cfg.AssumeRoleDuration = &d
 		}
 
@@ -668,7 +675,10 @@ func updateBool(dst *bool, section ini.Section, key string) {
 	if !section.Has(key) {
 		return
 	}
-	*dst = section.Bool(key)
+
+	// retains pre-(aws-sdk-go-v2#2276) behavior where non-bool value would resolve to false
+	v, _ := section.Bool(key)
+	*dst = v
 }
 
 // updateBoolPtr will only update the dst with the value in the section key,
@@ -677,8 +687,11 @@ func updateBoolPtr(dst **bool, section ini.Section, key string) {
 	if !section.Has(key) {
 		return
 	}
+
+	// retains pre-(aws-sdk-go-v2#2276) behavior where non-bool value would resolve to false
+	v, _ := section.Bool(key)
 	*dst = new(bool)
-	**dst = section.Bool(key)
+	**dst = v
 }
 
 // SharedConfigLoadError is an error for the shared config file failed to load.
@@ -805,7 +818,8 @@ func updateUseDualStackEndpoint(dst *endpoints.DualStackEndpointState, section i
 		return
 	}
 
-	if section.Bool(key) {
+	// retains pre-(aws-sdk-go-v2/#2276) behavior where non-bool value would resolve to false
+	if v, _ := section.Bool(key); v {
 		*dst = endpoints.DualStackEndpointStateEnabled
 	} else {
 		*dst = endpoints.DualStackEndpointStateDisabled
@@ -821,7 +835,8 @@ func updateUseFIPSEndpoint(dst *endpoints.FIPSEndpointState, section ini.Section
 		return
 	}
 
-	if section.Bool(key) {
+	// retains pre-(aws-sdk-go-v2/#2276) behavior where non-bool value would resolve to false
+	if v, _ := section.Bool(key); v {
 		*dst = endpoints.FIPSEndpointStateEnabled
 	} else {
 		*dst = endpoints.FIPSEndpointStateDisabled

vendor/github.com/aws/aws-sdk-go/aws/version.go

@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.44.333"
+const SDKVersion = "1.45.24"

vendor/github.com/aws/aws-sdk-go/internal/ini/literal_tokens.go

@@ -154,11 +154,11 @@ func (v ValueType) String() string {
 // ValueType enums
 const (
 	NoneType = ValueType(iota)
-	DecimalType
-	IntegerType
+	DecimalType // deprecated
+	IntegerType // deprecated
 	StringType
 	QuotedStringType
-	BoolType
+	BoolType // deprecated
 )
 
 // Value is a union container
@@ -166,9 +166,9 @@ type Value struct {
 	Type ValueType
 	raw  []rune
 
-	integer int64
-	decimal float64
-	boolean bool
+	integer int64 // deprecated
+	decimal float64 // deprecated
+	boolean bool // deprecated
 	str     string
 }
 
@@ -253,24 +253,6 @@ func newLitToken(b []rune) (Token, int, error) {
 		}
 
 		token = newToken(TokenLit, b[:n], QuotedStringType)
-	} else if isNumberValue(b) {
-		var base int
-		base, n, err = getNumericalValue(b)
-		if err != nil {
-			return token, 0, err
-		}
-
-		value := b[:n]
-		vType := IntegerType
-		if contains(value, '.') || hasExponent(value) {
-			vType = DecimalType
-		}
-		token = newToken(TokenLit, value, vType)
-		token.base = base
-	} else if isBoolValue(b) {
-		n, err = getBoolValue(b)
-
-		token = newToken(TokenLit, b[:n], BoolType)
 	} else {
 		n, err = getValue(b)
 		token = newToken(TokenLit, b[:n], StringType)
@@ -280,18 +262,33 @@ func newLitToken(b []rune) (Token, int, error) {
 }
 
 // IntValue returns an integer value
-func (v Value) IntValue() int64 {
-	return v.integer
+func (v Value) IntValue() (int64, bool) {
+	i, err := strconv.ParseInt(string(v.raw), 0, 64)
+	if err != nil {
+		return 0, false
+	}
+	return i, true
 }
 
 // FloatValue returns a float value
-func (v Value) FloatValue() float64 {
-	return v.decimal
+func (v Value) FloatValue() (float64, bool) {
+	f, err := strconv.ParseFloat(string(v.raw), 64)
+	if err != nil {
+		return 0, false
+	}
+	return f, true
 }
 
 // BoolValue returns a bool value
-func (v Value) BoolValue() bool {
-	return v.boolean
+func (v Value) BoolValue() (bool, bool) {
+	// we don't use ParseBool as it recognizes more than what we've
+	// historically supported
+	if isCaselessLitValue(runesTrue, v.raw) {
+		return true, true
+	} else if isCaselessLitValue(runesFalse, v.raw) {
+		return false, true
+	}
+	return false, false
 }
 
 func isTrimmable(r rune) bool {

vendor/github.com/aws/aws-sdk-go/internal/ini/visitor.go

@@ -145,17 +145,17 @@ func (t Section) ValueType(k string) (ValueType, bool) {
 }
 
 // Bool returns a bool value at k
-func (t Section) Bool(k string) bool {
+func (t Section) Bool(k string) (bool, bool) {
 	return t.values[k].BoolValue()
 }
 
 // Int returns an integer value at k
-func (t Section) Int(k string) int64 {
+func (t Section) Int(k string) (int64, bool) {
 	return t.values[k].IntValue()
 }
 
 // Float64 returns a float value at k
-func (t Section) Float64(k string) float64 {
+func (t Section) Float64(k string) (float64, bool) {
 	return t.values[k].FloatValue()
 }
 

vendor/github.com/aws/smithy-go/CHANGELOG.md

@@ -1,3 +1,9 @@
+# Release (2023-10-06)
+
+## Module Highlights
+* `github.com/aws/smithy-go`: v1.15.0
+  * **Feature**: Add `http.WithHeaderComment` middleware.
+
 # Release (2023-08-18)
 
 * No change notes available for this release.

vendor/github.com/aws/smithy-go/README.md

@@ -6,6 +6,21 @@
 
 **WARNING: All interfaces are subject to change.**
 
+## Can I use this?
+
+In order to generate a usable smithy client you must provide a [protocol definition](https://github.com/aws/smithy-go/blob/main/codegen/smithy-go-codegen/src/main/java/software/amazon/smithy/go/codegen/integration/ProtocolGenerator.java),
+such as [AWS restJson1](https://smithy.io/2.0/aws/protocols/aws-restjson1-protocol.html),
+in order to generate transport mechanisms and serialization/deserialization
+code ("serde") accordingly.
+
+The code generator does not currently support any protocols out of the box,
+therefore the useability of this project on its own is currently limited.
+Support for all [AWS protocols](https://smithy.io/2.0/aws/protocols/index.html)
+exists in [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2). We are
+tracking the movement of those out of the SDK into smithy-go in
+[#458](https://github.com/aws/smithy-go/issues/458), but there's currently no
+timeline for doing so.
+
 ## License
 
 This project is licensed under the Apache-2.0 License.

vendor/github.com/aws/smithy-go/go_module_metadata.go

@@ -3,4 +3,4 @@
 package smithy
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.14.2"
+const goModuleVersion = "1.15.0"

vendor/github.com/aws/smithy-go/transport/http/middleware_header_comment.go

@@ -0,0 +1,81 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/aws/smithy-go/middleware"
+)
+
+// WithHeaderComment instruments a middleware stack to append an HTTP field
+// comment to the given header as specified in RFC 9110
+// (https://www.rfc-editor.org/rfc/rfc9110#name-comments).
+//
+// The header is case-insensitive. If the provided header exists when the
+// middleware runs, the content will be inserted as-is enclosed in parentheses.
+//
+// Note that per the HTTP specification, comments are only allowed in fields
+// containing "comment" as part of their field value definition, but this API
+// will NOT verify whether the provided header is one of them.
+//
+// WithHeaderComment MAY be applied more than once to a middleware stack and/or
+// more than once per header.
+func WithHeaderComment(header, content string) func(*middleware.Stack) error {
+	return func(s *middleware.Stack) error {
+		m, err := getOrAddHeaderComment(s)
+		if err != nil {
+			return fmt.Errorf("get or add header comment: %v", err)
+		}
+
+		m.values.Add(header, content)
+		return nil
+	}
+}
+
+type headerCommentMiddleware struct {
+	values http.Header // hijack case-insensitive access APIs
+}
+
+func (*headerCommentMiddleware) ID() string {
+	return "headerComment"
+}
+
+func (m *headerCommentMiddleware) HandleBuild(ctx context.Context, in middleware.BuildInput, next middleware.BuildHandler) (
+	out middleware.BuildOutput, metadata middleware.Metadata, err error,
+) {
+	r, ok := in.Request.(*Request)
+	if !ok {
+		return out, metadata, fmt.Errorf("unknown transport type %T", in.Request)
+	}
+
+	for h, contents := range m.values {
+		for _, c := range contents {
+			if existing := r.Header.Get(h); existing != "" {
+				r.Header.Set(h, fmt.Sprintf("%s (%s)", existing, c))
+			}
+		}
+	}
+
+	return next.HandleBuild(ctx, in)
+}
+
+func getOrAddHeaderComment(s *middleware.Stack) (*headerCommentMiddleware, error) {
+	id := (*headerCommentMiddleware)(nil).ID()
+	m, ok := s.Build.Get(id)
+	if !ok {
+		m := &headerCommentMiddleware{values: http.Header{}}
+		if err := s.Build.Add(m, middleware.After); err != nil {
+			return nil, fmt.Errorf("add build: %v", err)
+		}
+
+		return m, nil
+	}
+
+	hc, ok := m.(*headerCommentMiddleware)
+	if !ok {
+		return nil, fmt.Errorf("existing middleware w/ id %s is not *headerCommentMiddleware", id)
+	}
+
+	return hc, nil
+}

vendor/github.com/emicklei/go-restful/v3/CHANGES.md

@@ -1,10 +1,21 @@
 # Change history of go-restful
 
-## [v3.9.0] - 20221-07-21
+## [v3.10.1] - 2022-11-19
+
+- fix broken 3.10.0 by using path package for joining paths
+
+## [v3.10.0] - 2022-10-11 - BROKEN
+
+- changed tokenizer to match std route match behavior; do not trimright the path (#511)
+- Add MIME_ZIP (#512)
+- Add MIME_ZIP and HEADER_ContentDisposition (#513)
+- Changed how to get query parameter issue #510
+
+## [v3.9.0] - 2022-07-21
 
 - add support for http.Handler implementations to work as FilterFunction, issue #504 (thanks to https://github.com/ggicci)
 
-## [v3.8.0] - 20221-06-06
+## [v3.8.0] - 2022-06-06
 
 - use exact matching of allowed domain entries, issue #489 (#493)
 	- this changes fixes [security] Authorization Bypass Through User-Controlled Key

vendor/github.com/emicklei/go-restful/v3/constants.go

@@ -7,12 +7,14 @@ package restful
 const (
 	MIME_XML   = "application/xml"          // Accept or Content-Type used in Consumes() and/or Produces()
 	MIME_JSON  = "application/json"         // Accept or Content-Type used in Consumes() and/or Produces()
+	MIME_ZIP   = "application/zip"          // Accept or Content-Type used in Consumes() and/or Produces()
 	MIME_OCTET = "application/octet-stream" // If Content-Type is not present in request, use the default
 
 	HEADER_Allow                         = "Allow"
 	HEADER_Accept                        = "Accept"
 	HEADER_Origin                        = "Origin"
 	HEADER_ContentType                   = "Content-Type"
+	HEADER_ContentDisposition            = "Content-Disposition"
 	HEADER_LastModified                  = "Last-Modified"
 	HEADER_AcceptEncoding                = "Accept-Encoding"
 	HEADER_ContentEncoding               = "Content-Encoding"

vendor/github.com/emicklei/go-restful/v3/request.go

@@ -31,7 +31,8 @@ func NewRequest(httpRequest *http.Request) *Request {
 // a "Unable to unmarshal content of type:" response is returned.
 // Valid values are restful.MIME_JSON and restful.MIME_XML
 // Example:
-// 	restful.DefaultRequestContentType(restful.MIME_JSON)
+//
+//	restful.DefaultRequestContentType(restful.MIME_JSON)
 func DefaultRequestContentType(mime string) {
 	defaultRequestContentType = mime
 }
@@ -48,7 +49,7 @@ func (r *Request) PathParameters() map[string]string {
 
 // QueryParameter returns the (first) Query parameter value by its name
 func (r *Request) QueryParameter(name string) string {
-	return r.Request.FormValue(name)
+	return r.Request.URL.Query().Get(name)
 }
 
 // QueryParameters returns the all the query parameters values by name

vendor/github.com/emicklei/go-restful/v3/response.go

@@ -109,6 +109,9 @@ func (r *Response) EntityWriter() (EntityReaderWriter, bool) {
 		if DefaultResponseMimeType == MIME_XML {
 			return entityAccessRegistry.accessorAt(MIME_XML)
 		}
+		if DefaultResponseMimeType == MIME_ZIP {
+			return entityAccessRegistry.accessorAt(MIME_ZIP)
+		}
 		// Fallback to whatever the route says it can produce.
 		// https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html
 		for _, each := range r.routeProduces {

vendor/github.com/emicklei/go-restful/v3/route.go

@@ -164,7 +164,7 @@ func tokenizePath(path string) []string {
 	if "/" == path {
 		return nil
 	}
-	return strings.Split(strings.Trim(path, "/"), "/")
+	return strings.Split(strings.TrimLeft(path, "/"), "/")
 }
 
 // for debugging
@@ -176,3 +176,5 @@ func (r *Route) String() string {
 func (r *Route) EnableContentEncoding(enabled bool) {
 	r.contentEncodingEnabled = &enabled
 }
+
+var TrimRightSlashEnabled = false

vendor/github.com/emicklei/go-restful/v3/route_builder.go

@@ -7,6 +7,7 @@ package restful
 import (
 	"fmt"
 	"os"
+	"path"
 	"reflect"
 	"runtime"
 	"strings"
@@ -46,11 +47,12 @@ type RouteBuilder struct {
 // Do evaluates each argument with the RouteBuilder itself.
 // This allows you to follow DRY principles without breaking the fluent programming style.
 // Example:
-// 		ws.Route(ws.DELETE("/{name}").To(t.deletePerson).Do(Returns200, Returns500))
 //
-//		func Returns500(b *RouteBuilder) {
-//			b.Returns(500, "Internal Server Error", restful.ServiceError{})
-//		}
+//	ws.Route(ws.DELETE("/{name}").To(t.deletePerson).Do(Returns200, Returns500))
+//
+//	func Returns500(b *RouteBuilder) {
+//		b.Returns(500, "Internal Server Error", restful.ServiceError{})
+//	}
 func (b *RouteBuilder) Do(oneArgBlocks ...func(*RouteBuilder)) *RouteBuilder {
 	for _, each := range oneArgBlocks {
 		each(b)
@@ -352,7 +354,7 @@ func (b *RouteBuilder) Build() Route {
 }
 
 func concatPath(path1, path2 string) string {
-	return strings.TrimRight(path1, "/") + "/" + strings.TrimLeft(path2, "/")
+	return path.Join(path1, path2)
 }
 
 var anonymousFuncCount int32

vendor/github.com/gemalto/kmip-go/.golangci.yml

@@ -35,15 +35,59 @@ linters-settings:
     # minimal occurrences count to trigger, 3 by default
     min-occurrences: 3
   depguard:
-    list-type: blacklist
-    include-go-root: false
-    packages:
-      - github.com/magiconair/properties/assert
-      - gopkg.in/go-playground/assert.v1
-      - github.com/pborman/uuid #replace with github.com/google/uuid
-    inTests:
-      - github.com/davecgh/go-spew/spew
-      - github.com/stretchr/testify
+    # Rules to apply.
+    #
+    # Variables:
+    # - File Variables
+    #   you can still use and exclamation mark ! in front of a variable to say not to use it.
+    #   Example !$test will match any file that is not a go test file.
+    #
+    #   `$all` - matches all go files
+    #   `$test` - matches all go test files
+    #
+    # - Package Variables
+    #
+    #  `$gostd` - matches all of go's standard library (Pulled from `GOROOT`)
+    #
+    # Default: Only allow $gostd in all files.
+    rules:
+      # Name of a rule.
+      all:
+        # List of file globs that will match this list of settings to compare against.
+        # Default: $all
+        files:
+          - $all
+        # List of allowed packages.
+        # allow:
+        #  - $gostd
+        # Packages that are not allowed where the value is a suggestion.
+        deny:
+          - pkg: github.com/magiconair/properties/assert
+            desc: Use testify/assert package instead
+          - pkg: gopkg.in/go-playground/assert.v1
+            desc: Use testify/assert package instead
+          - pkg: github.com/pborman/uuid
+            desc: Use google/uuid package instead
+      main:
+        files:
+          - "!$test"
+          # todo need to check the usage
+          - "!**authorization/conditions.go"
+          - "!**yugotest/assertions.go"
+          - "!**yugometrics/backendtesting/compliance.go"
+          - "!**scopes/auth_scope.go"
+        deny:
+          - pkg: github.com/davecgh/go-spew/spew
+            desc: spew is usually only used in tests
+          - pkg: github.com/stretchr/testify
+            desc: testify is usually only used in tests
+  gomodguard:
+    blocked:
+      modules:
+        - gopkg.in/go-playground/assert.v1:
+            recommendations:
+              - github.com/stretchr/testify
+            reason: "testify is the test assertion framework we use"
   misspell:
     # Correct spellings using locale preferences for US or UK.
     # Default is to use a neutral variety of English.
@@ -70,15 +114,11 @@ linters:
   enable:
     # default linters
     - staticcheck
-    - deadcode
     - errcheck
     - gosimple
     - govet
     - ineffassign
-    - structcheck
     - unused
-    - varcheck
-
     # additional linters
     - asciicheck
     - bidichk
@@ -115,7 +155,7 @@ linters:
 ##    - goimports           # checks import order.  We're not using goimports
 #    - gomnd                # too aggressive
     - gomoddirectives
-#    - gomodguard
+    - gomodguard
     - goprintffuncname
     - gosec
     - grouper

vendor/github.com/google/cel-go/checker/cost.go

@@ -533,14 +533,34 @@ func (c *coster) functionCost(function, overloadID string, target *AstNode, args
 
 	if est := c.estimator.EstimateCallCost(function, overloadID, target, args); est != nil {
 		callEst := *est
-		return CallEstimate{CostEstimate: callEst.Add(argCostSum())}
+		return CallEstimate{CostEstimate: callEst.Add(argCostSum()), ResultSize: est.ResultSize}
 	}
 	switch overloadID {
 	// O(n) functions
-	case overloads.StartsWithString, overloads.EndsWithString, overloads.StringToBytes, overloads.BytesToString, overloads.ExtQuoteString, overloads.ExtFormatString:
-		if overloadID == overloads.ExtFormatString {
+	case overloads.ExtFormatString:
+		if target != nil {
+			// ResultSize not calculated because we can't bound the max size.
 			return CallEstimate{CostEstimate: c.sizeEstimate(*target).MultiplyByCostFactor(common.StringTraversalCostFactor).Add(argCostSum())}
 		}
+	case overloads.StringToBytes:
+		if len(args) == 1 {
+			sz := c.sizeEstimate(args[0])
+			// ResultSize max is when each char converts to 4 bytes.
+			return CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor).Add(argCostSum()), ResultSize: &SizeEstimate{Min: sz.Min, Max: sz.Max * 4}}
+		}
+	case overloads.BytesToString:
+		if len(args) == 1 {
+			sz := c.sizeEstimate(args[0])
+			// ResultSize min is when 4 bytes convert to 1 char.
+			return CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor).Add(argCostSum()), ResultSize: &SizeEstimate{Min: sz.Min / 4, Max: sz.Max}}
+		}
+	case overloads.ExtQuoteString:
+		if len(args) == 1 {
+			sz := c.sizeEstimate(args[0])
+			// ResultSize max is when each char is escaped. 2 quote chars always added.
+			return CallEstimate{CostEstimate: sz.MultiplyByCostFactor(common.StringTraversalCostFactor).Add(argCostSum()), ResultSize: &SizeEstimate{Min: sz.Min + 2, Max: sz.Max*2 + 2}}
+		}
+	case overloads.StartsWithString, overloads.EndsWithString:
 		if len(args) == 1 {
 			return CallEstimate{CostEstimate: c.sizeEstimate(args[0]).MultiplyByCostFactor(common.StringTraversalCostFactor).Add(argCostSum())}
 		}

vendor/github.com/hashicorp/vault/api/README.md

@@ -4,6 +4,6 @@ Vault API
 This provides the `github.com/hashicorp/vault/api` package which contains code useful for interacting with a Vault server.
 
 For examples of how to use this module, see the [vault-examples](https://github.com/hashicorp/vault-examples) repo.
-For a step-by-step walkthrough on using these client libraries, see the [developer quickstart](https://www.vaultproject.io/docs/get-started/developer-qs).
+For a step-by-step walkthrough on using these client libraries, see the [developer quickstart](https://developer.hashicorp.com/vault/docs/get-started/developer-qs).
 
 [![GoDoc](https://godoc.org/github.com/hashicorp/vault/api?status.png)](https://godoc.org/github.com/hashicorp/vault/api)

vendor/github.com/hashicorp/vault/api/client.go

@@ -185,6 +185,9 @@ type Config struct {
 	// CloneToken from parent.
 	CloneToken bool
 
+	// CloneTLSConfig from parent (tls.Config).
+	CloneTLSConfig bool
+
 	// ReadYourWrites ensures isolated read-after-write semantics by
 	// providing discovered cluster replication states in each request.
 	// The shared state is automatically propagated to all Client clones.
@@ -290,7 +293,14 @@ func (c *Config) configureTLS(t *TLSConfig) error {
 	if c.HttpClient == nil {
 		c.HttpClient = DefaultConfig().HttpClient
 	}
-	clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig
+
+	transport, ok := c.HttpClient.Transport.(*http.Transport)
+	if !ok {
+		return fmt.Errorf(
+			"unsupported HTTPClient transport type %T", c.HttpClient.Transport)
+	}
+
+	clientTLSConfig := transport.TLSClientConfig
 
 	var clientCert tls.Certificate
 	foundClientCert := false
@@ -535,7 +545,7 @@ func (c *Config) ParseAddress(address string) (*url.URL, error) {
 			// be pointing to the protocol used in the application layer and not to
 			// the transport layer. Hence, setting the fields accordingly.
 			u.Scheme = "http"
-			u.Host = socket
+			u.Host = "localhost"
 			u.Path = ""
 		} else {
 			return nil, fmt.Errorf("attempting to specify unix:// address with non-transport transport")
@@ -988,7 +998,9 @@ func (c *Client) Namespace() string {
 func (c *Client) WithNamespace(namespace string) *Client {
 	c2 := *c
 	c2.modifyLock = sync.RWMutex{}
-	c2.headers = c.Headers()
+	c.modifyLock.RLock()
+	c2.headers = c.headersInternal()
+	c.modifyLock.RUnlock()
 	if namespace == "" {
 		c2.ClearNamespace()
 	} else {
@@ -1025,7 +1037,12 @@ func (c *Client) ClearToken() {
 func (c *Client) Headers() http.Header {
 	c.modifyLock.RLock()
 	defer c.modifyLock.RUnlock()
+	return c.headersInternal()
+}
 
+// headersInternal gets the current set of headers used for requests. Must be called
+// with the read modifyLock held.
+func (c *Client) headersInternal() http.Header {
 	if c.headers == nil {
 		return nil
 	}
@@ -1143,6 +1160,26 @@ func (c *Client) ReadYourWrites() bool {
 	return c.config.ReadYourWrites
 }
 
+// SetCloneTLSConfig from parent.
+func (c *Client) SetCloneTLSConfig(clone bool) {
+	c.modifyLock.Lock()
+	defer c.modifyLock.Unlock()
+	c.config.modifyLock.Lock()
+	defer c.config.modifyLock.Unlock()
+
+	c.config.CloneTLSConfig = clone
+}
+
+// CloneTLSConfig gets the configured CloneTLSConfig value.
+func (c *Client) CloneTLSConfig() bool {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
+
+	return c.config.CloneTLSConfig
+}
+
 // Clone creates a new client with the same configuration. Note that the same
 // underlying http.Client is used; modifying the client from more than one
 // goroutine at once may not be safe, so modify the client as needed and then
@@ -1153,24 +1190,28 @@ func (c *Client) ReadYourWrites() bool {
 // the api.Config struct, such as policy override and wrapping function
 // behavior, must currently then be set as desired on the new client.
 func (c *Client) Clone() (*Client, error) {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
 	return c.clone(c.config.CloneHeaders)
 }
 
 // CloneWithHeaders creates a new client similar to Clone, with the difference
-// being that the  headers are always cloned
+// being that the headers are always cloned
 func (c *Client) CloneWithHeaders() (*Client, error) {
+	c.modifyLock.RLock()
+	defer c.modifyLock.RUnlock()
+	c.config.modifyLock.RLock()
+	defer c.config.modifyLock.RUnlock()
 	return c.clone(true)
 }
 
 // clone creates a new client, with the headers being cloned based on the
-// passed in cloneheaders boolean
+// passed in cloneheaders boolean.
+// Must be called with the read lock and config read lock held.
 func (c *Client) clone(cloneHeaders bool) (*Client, error) {
-	c.modifyLock.RLock()
-	defer c.modifyLock.RUnlock()
-
 	config := c.config
-	config.modifyLock.RLock()
-	defer config.modifyLock.RUnlock()
 
 	newConfig := &Config{
 		Address:        config.Address,
@@ -1189,13 +1230,18 @@ func (c *Client) clone(cloneHeaders bool) (*Client, error) {
 		CloneToken:     config.CloneToken,
 		ReadYourWrites: config.ReadYourWrites,
 	}
+
+	if config.CloneTLSConfig {
+		newConfig.clientTLSConfig = config.clientTLSConfig
+	}
+
 	client, err := NewClient(newConfig)
 	if err != nil {
 		return nil, err
 	}
 
 	if cloneHeaders {
-		client.SetHeaders(c.Headers().Clone())
+		client.SetHeaders(c.headersInternal().Clone())
 	}
 
 	if config.CloneToken {
@@ -1226,6 +1272,7 @@ func (c *Client) NewRequest(method, requestPath string) *Request {
 	mfaCreds := c.mfaCreds
 	wrappingLookupFunc := c.wrappingLookupFunc
 	policyOverride := c.policyOverride
+	headers := c.headersInternal()
 	c.modifyLock.RUnlock()
 
 	host := addr.Host
@@ -1270,7 +1317,7 @@ func (c *Client) NewRequest(method, requestPath string) *Request {
 		req.WrapTTL = DefaultWrappingLookupFunc(method, lookupPath)
 	}
 
-	req.Headers = c.Headers()
+	req.Headers = headers
 	req.PolicyOverride = policyOverride
 
 	return req

vendor/github.com/hashicorp/vault/api/kv.go

@@ -38,7 +38,7 @@ type KVSecret struct {
 // by default when a server is started in -dev mode. See the kvv2 struct.
 //
 // Learn more about the KV secrets engine here:
-// https://www.vaultproject.io/docs/secrets/kv
+// https://developer.hashicorp.com/vault/docs/secrets/kv
 func (c *Client) KVv1(mountPath string) *KVv1 {
 	return &KVv1{c: c, mountPath: mountPath}
 }
@@ -53,7 +53,7 @@ func (c *Client) KVv1(mountPath string) *KVv1 {
 // as these are the default settings when a server is started in -dev mode.
 //
 // Learn more about the KV secrets engine here:
-// https://www.vaultproject.io/docs/secrets/kv
+// https://developer.hashicorp.com/vault/docs/secrets/kv
 func (c *Client) KVv2(mountPath string) *KVv2 {
 	return &KVv2{c: c, mountPath: mountPath}
 }

vendor/github.com/hashicorp/vault/api/plugin_helpers.go

@@ -12,13 +12,23 @@ import (
 	"flag"
 	"net/url"
 	"os"
-	"regexp"
 
 	"github.com/go-jose/go-jose/v3/jwt"
 
 	"github.com/hashicorp/errwrap"
 )
 
+// This file contains helper code used when writing Vault auth method or secrets engine plugins.
+//
+// As such, it would be better located in the sdk module with the rest of the code which is only to support plugins,
+// rather than api, but is here for historical reasons. (The api module used to depend on the sdk module, this code
+// calls NewClient within the api package, so placing it in the sdk would have created a dependency cycle. This reason
+// is now historical, as the dependency between sdk and api has since been reversed in direction.)
+// Moving this code to the sdk would be appropriate if an api v2.0.0 release is ever planned.
+//
+// This helper code is used when a plugin is hosted by Vault 1.11 and earlier. Vault 1.12 and sdk v0.6.0 introduced
+// version 5 of the backend plugin interface, which uses go-plugin's AutoMTLS feature instead of this code.
+
 const (
 	// PluginAutoMTLSEnv is used to ensure AutoMTLS is used. This will override
 	// setting a TLSProviderFunc for a plugin.
@@ -33,50 +43,6 @@ const (
 	PluginUnwrapTokenEnv = "VAULT_UNWRAP_TOKEN"
 )
 
-// sudoPaths is a map containing the paths that require a token's policy
-// to have the "sudo" capability. The keys are the paths as strings, in
-// the same format as they are returned by the OpenAPI spec. The values
-// are the regular expressions that can be used to test whether a given
-// path matches that path or not (useful specifically for the paths that
-// contain templated fields.)
-var sudoPaths = map[string]*regexp.Regexp{
-	"/auth/token/accessors/":                        regexp.MustCompile(`^/auth/token/accessors/?$`),
-	"/pki/root":                                     regexp.MustCompile(`^/pki/root$`),
-	"/pki/root/sign-self-issued":                    regexp.MustCompile(`^/pki/root/sign-self-issued$`),
-	"/sys/audit":                                    regexp.MustCompile(`^/sys/audit$`),
-	"/sys/audit/{path}":                             regexp.MustCompile(`^/sys/audit/.+$`),
-	"/sys/auth/{path}":                              regexp.MustCompile(`^/sys/auth/.+$`),
-	"/sys/auth/{path}/tune":                         regexp.MustCompile(`^/sys/auth/.+/tune$`),
-	"/sys/config/auditing/request-headers":          regexp.MustCompile(`^/sys/config/auditing/request-headers$`),
-	"/sys/config/auditing/request-headers/{header}": regexp.MustCompile(`^/sys/config/auditing/request-headers/.+$`),
-	"/sys/config/cors":                              regexp.MustCompile(`^/sys/config/cors$`),
-	"/sys/config/ui/headers/":                       regexp.MustCompile(`^/sys/config/ui/headers/?$`),
-	"/sys/config/ui/headers/{header}":               regexp.MustCompile(`^/sys/config/ui/headers/.+$`),
-	"/sys/leases":                                   regexp.MustCompile(`^/sys/leases$`),
-	"/sys/leases/lookup/":                           regexp.MustCompile(`^/sys/leases/lookup/?$`),
-	"/sys/leases/lookup/{prefix}":                   regexp.MustCompile(`^/sys/leases/lookup/.+$`),
-	"/sys/leases/revoke-force/{prefix}":             regexp.MustCompile(`^/sys/leases/revoke-force/.+$`),
-	"/sys/leases/revoke-prefix/{prefix}":            regexp.MustCompile(`^/sys/leases/revoke-prefix/.+$`),
-	"/sys/plugins/catalog/{name}":                   regexp.MustCompile(`^/sys/plugins/catalog/[^/]+$`),
-	"/sys/plugins/catalog/{type}":                   regexp.MustCompile(`^/sys/plugins/catalog/[\w-]+$`),
-	"/sys/plugins/catalog/{type}/{name}":            regexp.MustCompile(`^/sys/plugins/catalog/[\w-]+/[^/]+$`),
-	"/sys/raw":                                      regexp.MustCompile(`^/sys/raw$`),
-	"/sys/raw/{path}":                               regexp.MustCompile(`^/sys/raw/.+$`),
-	"/sys/remount":                                  regexp.MustCompile(`^/sys/remount$`),
-	"/sys/revoke-force/{prefix}":                    regexp.MustCompile(`^/sys/revoke-force/.+$`),
-	"/sys/revoke-prefix/{prefix}":                   regexp.MustCompile(`^/sys/revoke-prefix/.+$`),
-	"/sys/rotate":                                   regexp.MustCompile(`^/sys/rotate$`),
-	"/sys/internal/inspect/router/{tag}":            regexp.MustCompile(`^/sys/internal/inspect/router/.+$`),
-
-	// enterprise-only paths
-	"/sys/replication/dr/primary/secondary-token":          regexp.MustCompile(`^/sys/replication/dr/primary/secondary-token$`),
-	"/sys/replication/performance/primary/secondary-token": regexp.MustCompile(`^/sys/replication/performance/primary/secondary-token$`),
-	"/sys/replication/primary/secondary-token":             regexp.MustCompile(`^/sys/replication/primary/secondary-token$`),
-	"/sys/replication/reindex":                             regexp.MustCompile(`^/sys/replication/reindex$`),
-	"/sys/storage/raft/snapshot-auto/config/":              regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/?$`),
-	"/sys/storage/raft/snapshot-auto/config/{name}":        regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/[^/]+$`),
-}
-
 // PluginAPIClientMeta is a helper that plugins can use to configure TLS connections
 // back to Vault.
 type PluginAPIClientMeta struct {
@@ -244,28 +210,3 @@ func VaultPluginTLSProviderContext(ctx context.Context, apiTLSConfig *TLSConfig)
 		return tlsConfig, nil
 	}
 }
-
-func SudoPaths() map[string]*regexp.Regexp {
-	return sudoPaths
-}
-
-// Determine whether the given path requires the sudo capability
-func IsSudoPath(path string) bool {
-	// Return early if the path is any of the non-templated sudo paths.
-	if _, ok := sudoPaths[path]; ok {
-		return true
-	}
-
-	// Some sudo paths have templated fields in them.
-	// (e.g. /sys/revoke-prefix/{prefix})
-	// The values in the sudoPaths map are actually regular expressions,
-	// so we can check if our path matches against them.
-	for _, sudoPathRegexp := range sudoPaths {
-		match := sudoPathRegexp.MatchString(path)
-		if match {
-			return true
-		}
-	}
-
-	return false
-}

vendor/github.com/hashicorp/vault/api/plugin_runtime_types.go

@@ -0,0 +1,41 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+// NOTE: this file was copied from
+// https://github.com/hashicorp/vault/blob/main/sdk/helper/consts/plugin_runtime_types.go
+// Any changes made should be made to both files at the same time.
+
+import "fmt"
+
+var PluginRuntimeTypes = []PluginRuntimeType{
+	PluginRuntimeTypeUnsupported,
+	PluginRuntimeTypeContainer,
+}
+
+type PluginRuntimeType uint32
+
+// This is a list of PluginRuntimeTypes used by Vault.
+const (
+	PluginRuntimeTypeUnsupported PluginRuntimeType = iota
+	PluginRuntimeTypeContainer
+)
+
+func (r PluginRuntimeType) String() string {
+	switch r {
+	case PluginRuntimeTypeContainer:
+		return "container"
+	default:
+		return "unsupported"
+	}
+}
+
+func ParsePluginRuntimeType(PluginRuntimeType string) (PluginRuntimeType, error) {
+	switch PluginRuntimeType {
+	case "container":
+		return PluginRuntimeTypeContainer, nil
+	default:
+		return PluginRuntimeTypeUnsupported, fmt.Errorf("%q is not a supported plugin runtime type", PluginRuntimeType)
+	}
+}

vendor/github.com/hashicorp/vault/api/replication_status.go

@@ -0,0 +1,130 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+const (
+	apiRepPerformanceStatusPath = "/v1/sys/replication/performance/status"
+	apiRepDRStatusPath          = "/v1/sys/replication/dr/status"
+	apiRepStatusPath            = "/v1/sys/replication/status"
+)
+
+type ClusterInfo struct {
+	APIAddr          string `json:"api_address,omitempty" mapstructure:"api_address"`
+	ClusterAddress   string `json:"cluster_address,omitempty" mapstructure:"cluster_address"`
+	ConnectionStatus string `json:"connection_status,omitempty" mapstructure:"connection_status"`
+	LastHeartBeat    string `json:"last_heartbeat,omitempty" mapstructure:"last_heartbeat"`
+	NodeID           string `json:"node_id,omitempty" mapstructure:"node_id"`
+}
+
+type ReplicationStatusGenericResponse struct {
+	LastDRWAL             uint64 `json:"last_dr_wal,omitempty" mapstructure:"last_dr_wal"`
+	LastReindexEpoch      string `json:"last_reindex_epoch,omitempty" mapstructure:"last_reindex_epoch"`
+	ClusterID             string `json:"cluster_id,omitempty" mapstructure:"cluster_id"`
+	LastWAL               uint64 `json:"last_wal,omitempty" mapstructure:"last_wal"`
+	MerkleRoot            string `json:"merkle_root,omitempty" mapstructure:"merkle_root"`
+	Mode                  string `json:"mode,omitempty" mapstructure:"mode"`
+	PrimaryClusterAddr    string `json:"primary_cluster_addr,omitempty" mapstructure:"primary_cluster_addr"`
+	LastPerformanceWAL    uint64 `json:"last_performance_wal,omitempty" mapstructure:"last_performance_wal"`
+	State                 string `json:"state,omitempty" mapstructure:"state"`
+	LastRemoteWAL         uint64 `json:"last_remote_wal,omitempty" mapstructure:"last_remote_wal"`
+	SecondaryID           string `json:"secondary_id,omitempty" mapstructure:"secondary_id"`
+	SSCTGenerationCounter uint64 `json:"ssct_generation_counter,omitempty" mapstructure:"ssct_generation_counter"`
+
+	KnownSecondaries         []string      `json:"known_secondaries,omitempty" mapstructure:"known_secondaries"`
+	KnownPrimaryClusterAddrs []string      `json:"known_primary_cluster_addrs,omitempty" mapstructure:"known_primary_cluster_addrs"`
+	Primaries                []ClusterInfo `json:"primaries,omitempty" mapstructure:"primaries"`
+	Secondaries              []ClusterInfo `json:"secondaries,omitempty" mapstructure:"secondaries"`
+}
+
+type ReplicationStatusResponse struct {
+	DR          ReplicationStatusGenericResponse `json:"dr,omitempty" mapstructure:"dr"`
+	Performance ReplicationStatusGenericResponse `json:"performance,omitempty" mapstructure:"performance"`
+}
+
+func (c *Sys) ReplicationStatus() (*ReplicationStatusResponse, error) {
+	return c.ReplicationStatusWithContext(context.Background(), apiRepStatusPath)
+}
+
+func (c *Sys) ReplicationPerformanceStatusWithContext(ctx context.Context) (*ReplicationStatusGenericResponse, error) {
+	s, err := c.ReplicationStatusWithContext(ctx, apiRepPerformanceStatusPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &s.Performance, nil
+}
+
+func (c *Sys) ReplicationDRStatusWithContext(ctx context.Context) (*ReplicationStatusGenericResponse, error) {
+	s, err := c.ReplicationStatusWithContext(ctx, apiRepDRStatusPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &s.DR, nil
+}
+
+func (c *Sys) ReplicationStatusWithContext(ctx context.Context, path string) (*ReplicationStatusResponse, error) {
+	// default to replication/status
+	if path == "" {
+		path = apiRepStatusPath
+	}
+
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest(http.MethodGet, path)
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	// First decode response into a map[string]interface{}
+	data := make(map[string]interface{})
+	dec := json.NewDecoder(resp.Body)
+	dec.UseNumber()
+	if err := dec.Decode(&data); err != nil {
+		return nil, err
+	}
+
+	rawData, ok := data["data"]
+	if !ok {
+		return nil, fmt.Errorf("empty data in replication status response")
+	}
+
+	s := &ReplicationStatusResponse{}
+	g := &ReplicationStatusGenericResponse{}
+	switch {
+	case path == apiRepPerformanceStatusPath:
+		err = mapstructure.Decode(rawData, g)
+		if err != nil {
+			return nil, err
+		}
+		s.Performance = *g
+	case path == apiRepDRStatusPath:
+		err = mapstructure.Decode(rawData, g)
+		if err != nil {
+			return nil, err
+		}
+		s.DR = *g
+	default:
+		err = mapstructure.Decode(rawData, s)
+		if err != nil {
+			return nil, err
+		}
+		return s, err
+	}
+
+	return s, err
+}

vendor/github.com/hashicorp/vault/api/sudo_paths.go

@@ -0,0 +1,87 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+import (
+	"regexp"
+)
+
+// sudoPaths is a map containing the paths that require a token's policy
+// to have the "sudo" capability. The keys are the paths as strings, in
+// the same format as they are returned by the OpenAPI spec. The values
+// are the regular expressions that can be used to test whether a given
+// path matches that path or not (useful specifically for the paths that
+// contain templated fields.)
+var sudoPaths = map[string]*regexp.Regexp{
+	"/auth/token/accessors":                         regexp.MustCompile(`^/auth/token/accessors/?$`),
+	"/auth/token/revoke-orphan":                     regexp.MustCompile(`^/auth/token/revoke-orphan$`),
+	"/pki/root":                                     regexp.MustCompile(`^/pki/root$`),
+	"/pki/root/sign-self-issued":                    regexp.MustCompile(`^/pki/root/sign-self-issued$`),
+	"/sys/audit":                                    regexp.MustCompile(`^/sys/audit$`),
+	"/sys/audit/{path}":                             regexp.MustCompile(`^/sys/audit/.+$`),
+	"/sys/auth/{path}":                              regexp.MustCompile(`^/sys/auth/.+$`),
+	"/sys/auth/{path}/tune":                         regexp.MustCompile(`^/sys/auth/.+/tune$`),
+	"/sys/config/auditing/request-headers":          regexp.MustCompile(`^/sys/config/auditing/request-headers$`),
+	"/sys/config/auditing/request-headers/{header}": regexp.MustCompile(`^/sys/config/auditing/request-headers/.+$`),
+	"/sys/config/cors":                              regexp.MustCompile(`^/sys/config/cors$`),
+	"/sys/config/ui/headers":                        regexp.MustCompile(`^/sys/config/ui/headers/?$`),
+	"/sys/config/ui/headers/{header}":               regexp.MustCompile(`^/sys/config/ui/headers/.+$`),
+	"/sys/internal/inspect/router/{tag}":            regexp.MustCompile(`^/sys/internal/inspect/router/.+$`),
+	"/sys/leases":                                   regexp.MustCompile(`^/sys/leases$`),
+	// This entry is a bit wrong... sys/leases/lookup does NOT require sudo. But sys/leases/lookup/ with a trailing
+	// slash DOES require sudo. But the part of the Vault CLI that uses this logic doesn't pass operation-appropriate
+	// trailing slashes, it always strips them off, so we end up giving the wrong answer for one of these.
+	"/sys/leases/lookup/{prefix}":                 regexp.MustCompile(`^/sys/leases/lookup(?:/.+)?$`),
+	"/sys/leases/revoke-force/{prefix}":           regexp.MustCompile(`^/sys/leases/revoke-force/.+$`),
+	"/sys/leases/revoke-prefix/{prefix}":          regexp.MustCompile(`^/sys/leases/revoke-prefix/.+$`),
+	"/sys/plugins/catalog/{name}":                 regexp.MustCompile(`^/sys/plugins/catalog/[^/]+$`),
+	"/sys/plugins/catalog/{type}":                 regexp.MustCompile(`^/sys/plugins/catalog/[\w-]+$`),
+	"/sys/plugins/catalog/{type}/{name}":          regexp.MustCompile(`^/sys/plugins/catalog/[\w-]+/[^/]+$`),
+	"/sys/plugins/runtimes/catalog":               regexp.MustCompile(`^/sys/plugins/runtimes/catalog/?$`),
+	"/sys/plugins/runtimes/catalog/{type}/{name}": regexp.MustCompile(`^/sys/plugins/runtimes/catalog/[\w-]+/[^/]+$`),
+	"/sys/raw/{path}":                             regexp.MustCompile(`^/sys/raw(?:/.+)?$`),
+	"/sys/remount":                                regexp.MustCompile(`^/sys/remount$`),
+	"/sys/revoke-force/{prefix}":                  regexp.MustCompile(`^/sys/revoke-force/.+$`),
+	"/sys/revoke-prefix/{prefix}":                 regexp.MustCompile(`^/sys/revoke-prefix/.+$`),
+	"/sys/rotate":                                 regexp.MustCompile(`^/sys/rotate$`),
+	"/sys/seal":                                   regexp.MustCompile(`^/sys/seal$`),
+	"/sys/step-down":                              regexp.MustCompile(`^/sys/step-down$`),
+
+	// enterprise-only paths
+	"/sys/replication/dr/primary/secondary-token":          regexp.MustCompile(`^/sys/replication/dr/primary/secondary-token$`),
+	"/sys/replication/performance/primary/secondary-token": regexp.MustCompile(`^/sys/replication/performance/primary/secondary-token$`),
+	"/sys/replication/primary/secondary-token":             regexp.MustCompile(`^/sys/replication/primary/secondary-token$`),
+	"/sys/replication/reindex":                             regexp.MustCompile(`^/sys/replication/reindex$`),
+	"/sys/storage/raft/snapshot-auto/config":               regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/?$`),
+	"/sys/storage/raft/snapshot-auto/config/{name}":        regexp.MustCompile(`^/sys/storage/raft/snapshot-auto/config/[^/]+$`),
+}
+
+func SudoPaths() map[string]*regexp.Regexp {
+	return sudoPaths
+}
+
+// Determine whether the given path requires the sudo capability.
+// Note that this uses hardcoded static path information, so will return incorrect results for paths in namespaces,
+// or for secret engines mounted at non-default paths.
+// Expects to receive a path with an initial slash, but no trailing slashes, as the Vault CLI (the only known and
+// expected user of this function) sanitizes its paths that way.
+func IsSudoPath(path string) bool {
+	// Return early if the path is any of the non-templated sudo paths.
+	if _, ok := sudoPaths[path]; ok {
+		return true
+	}
+
+	// Some sudo paths have templated fields in them.
+	// (e.g. /sys/revoke-prefix/{prefix})
+	// The values in the sudoPaths map are actually regular expressions,
+	// so we can check if our path matches against them.
+	for _, sudoPathRegexp := range sudoPaths {
+		match := sudoPathRegexp.MatchString(path)
+		if match {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/hashicorp/vault/api/sys_plugins.go

@@ -144,6 +144,7 @@ type GetPluginResponse struct {
 	Args              []string `json:"args"`
 	Builtin           bool     `json:"builtin"`
 	Command           string   `json:"command"`
+	OCIImage          string   `json:"oci_image"`
 	Name              string   `json:"name"`
 	SHA256            string   `json:"sha256"`
 	DeprecationStatus string   `json:"deprecation_status,omitempty"`
@@ -201,6 +202,13 @@ type RegisterPluginInput struct {
 
 	// Version is the optional version of the plugin being registered
 	Version string `json:"version,omitempty"`
+
+	// OCIImage specifies the container image to run as a plugin.
+	OCIImage string `json:"oci_image,omitempty"`
+
+	// Env specifies a list of key=value pairs to add to the plugin's environment
+	// variables.
+	Env []string `json:"env,omitempty"`
 }
 
 // RegisterPlugin wraps RegisterPluginWithContext using context.Background.

vendor/github.com/hashicorp/vault/api/sys_plugins_runtimes.go

@@ -0,0 +1,189 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package api
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/mitchellh/mapstructure"
+)
+
+// GetPluginRuntimeInput is used as input to the GetPluginRuntime function.
+type GetPluginRuntimeInput struct {
+	Name string `json:"-"`
+
+	// Type of the plugin runtime. Required.
+	Type PluginRuntimeType `json:"type"`
+}
+
+// GetPluginRuntimeResponse is the response from the GetPluginRuntime call.
+type GetPluginRuntimeResponse struct {
+	Type         string `json:"type"`
+	Name         string `json:"name"`
+	OCIRuntime   string `json:"oci_runtime"`
+	CgroupParent string `json:"cgroup_parent"`
+	CPU          int64  `json:"cpu_nanos"`
+	Memory       int64  `json:"memory_bytes"`
+}
+
+// GetPluginRuntime retrieves information about the plugin.
+func (c *Sys) GetPluginRuntime(ctx context.Context, i *GetPluginRuntimeInput) (*GetPluginRuntimeResponse, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	path := pluginRuntimeCatalogPathByType(i.Type, i.Name)
+	req := c.c.NewRequest(http.MethodGet, path)
+
+	resp, err := c.c.rawRequestWithContext(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var result struct {
+		Data *GetPluginRuntimeResponse
+	}
+	err = resp.DecodeJSON(&result)
+	if err != nil {
+		return nil, err
+	}
+	return result.Data, err
+}
+
+// RegisterPluginRuntimeInput is used as input to the RegisterPluginRuntime function.
+type RegisterPluginRuntimeInput struct {
+	// Name is the name of the plugin. Required.
+	Name string `json:"-"`
+
+	// Type of the plugin. Required.
+	Type PluginRuntimeType `json:"type"`
+
+	OCIRuntime   string `json:"oci_runtime,omitempty"`
+	CgroupParent string `json:"cgroup_parent,omitempty"`
+	CPU          int64  `json:"cpu,omitempty"`
+	Memory       int64  `json:"memory,omitempty"`
+}
+
+// RegisterPluginRuntime registers the plugin with the given information.
+func (c *Sys) RegisterPluginRuntime(ctx context.Context, i *RegisterPluginRuntimeInput) error {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	path := pluginRuntimeCatalogPathByType(i.Type, i.Name)
+	req := c.c.NewRequest(http.MethodPut, path)
+
+	if err := req.SetJSONBody(i); err != nil {
+		return err
+	}
+
+	resp, err := c.c.rawRequestWithContext(ctx, req)
+	if err == nil {
+		defer resp.Body.Close()
+	}
+	return err
+}
+
+// DeregisterPluginRuntimeInput is used as input to the DeregisterPluginRuntime function.
+type DeregisterPluginRuntimeInput struct {
+	// Name is the name of the plugin runtime. Required.
+	Name string `json:"-"`
+
+	// Type of the plugin. Required.
+	Type PluginRuntimeType `json:"type"`
+}
+
+// DeregisterPluginRuntime removes the plugin with the given name from the plugin
+// catalog.
+func (c *Sys) DeregisterPluginRuntime(ctx context.Context, i *DeregisterPluginRuntimeInput) error {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	path := pluginRuntimeCatalogPathByType(i.Type, i.Name)
+	req := c.c.NewRequest(http.MethodDelete, path)
+	resp, err := c.c.rawRequestWithContext(ctx, req)
+	if err == nil {
+		defer resp.Body.Close()
+	}
+	return err
+}
+
+type PluginRuntimeDetails struct {
+	Type         string `json:"type" mapstructure:"type"`
+	Name         string `json:"name" mapstructure:"name"`
+	OCIRuntime   string `json:"oci_runtime" mapstructure:"oci_runtime"`
+	CgroupParent string `json:"cgroup_parent" mapstructure:"cgroup_parent"`
+	CPU          int64  `json:"cpu_nanos" mapstructure:"cpu_nanos"`
+	Memory       int64  `json:"memory_bytes" mapstructure:"memory_bytes"`
+}
+
+// ListPluginRuntimesInput is used as input to the ListPluginRuntimes function.
+type ListPluginRuntimesInput struct {
+	// Type of the plugin. Required.
+	Type PluginRuntimeType `json:"type"`
+}
+
+// ListPluginRuntimesResponse is the response from the ListPluginRuntimes call.
+type ListPluginRuntimesResponse struct {
+	// RuntimesByType is the list of plugin runtimes by type.
+	Runtimes []PluginRuntimeDetails `json:"runtimes"`
+}
+
+// ListPluginRuntimes lists all plugin runtimes in the catalog and returns their names as a
+// list of strings.
+func (c *Sys) ListPluginRuntimes(ctx context.Context, input *ListPluginRuntimesInput) (*ListPluginRuntimesResponse, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	if input != nil && input.Type == PluginRuntimeTypeUnsupported {
+		return nil, fmt.Errorf("%q is not a supported runtime type", input.Type.String())
+	}
+
+	resp, err := c.c.rawRequestWithContext(ctx, c.c.NewRequest(http.MethodGet, "/v1/sys/plugins/runtimes/catalog"))
+	if err != nil && resp == nil {
+		return nil, err
+	}
+	if resp == nil {
+		return nil, nil
+	}
+	defer resp.Body.Close()
+
+	secret, err := ParseSecret(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if secret == nil || secret.Data == nil {
+		return nil, errors.New("data from server response is empty")
+	}
+	if _, ok := secret.Data["runtimes"]; !ok {
+		return nil, fmt.Errorf("data from server response does not contain runtimes")
+	}
+
+	var runtimes []PluginRuntimeDetails
+	if err = mapstructure.Decode(secret.Data["runtimes"], &runtimes); err != nil {
+		return nil, err
+	}
+
+	// return all runtimes in the catalog
+	if input == nil {
+		return &ListPluginRuntimesResponse{Runtimes: runtimes}, nil
+	}
+
+	result := &ListPluginRuntimesResponse{
+		Runtimes: []PluginRuntimeDetails{},
+	}
+	for _, runtime := range runtimes {
+		if runtime.Type == input.Type.String() {
+			result.Runtimes = append(result.Runtimes, runtime)
+		}
+	}
+	return result, nil
+}
+
+// pluginRuntimeCatalogPathByType is a helper to construct the proper API path by plugin type
+func pluginRuntimeCatalogPathByType(runtimeType PluginRuntimeType, name string) string {
+	return fmt.Sprintf("/v1/sys/plugins/runtimes/catalog/%s/%s", runtimeType, name)
+}

vendor/github.com/hashicorp/vault/api/sys_raft.go

@@ -276,11 +276,19 @@ func (c *Sys) RaftAutopilotState() (*AutopilotState, error) {
 	return c.RaftAutopilotStateWithContext(context.Background())
 }
 
+// RaftAutopilotStateWithToken wraps RaftAutopilotStateWithContext using the given token.
+func (c *Sys) RaftAutopilotStateWithDRToken(drToken string) (*AutopilotState, error) {
+	return c.RaftAutopilotStateWithContext(context.WithValue(context.Background(), "dr-token", drToken))
+}
+
 // RaftAutopilotStateWithContext returns the state of the raft cluster as seen by autopilot.
 func (c *Sys) RaftAutopilotStateWithContext(ctx context.Context) (*AutopilotState, error) {
 	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
 	defer cancelFunc()
 
+	if ctx.Value("dr-token") != nil {
+		c.c.SetToken(ctx.Value("dr-token").(string))
+	}
 	r := c.c.NewRequest(http.MethodGet, "/v1/sys/storage/raft/autopilot/state")
 
 	resp, err := c.c.rawRequestWithContext(ctx, r)
@@ -316,11 +324,20 @@ func (c *Sys) RaftAutopilotConfiguration() (*AutopilotConfig, error) {
 	return c.RaftAutopilotConfigurationWithContext(context.Background())
 }
 
+// RaftAutopilotConfigurationWithDRToken wraps RaftAutopilotConfigurationWithContext using the given token.
+func (c *Sys) RaftAutopilotConfigurationWithDRToken(drToken string) (*AutopilotConfig, error) {
+	return c.RaftAutopilotConfigurationWithContext(context.WithValue(context.Background(), "dr-token", drToken))
+}
+
 // RaftAutopilotConfigurationWithContext fetches the autopilot config.
 func (c *Sys) RaftAutopilotConfigurationWithContext(ctx context.Context) (*AutopilotConfig, error) {
 	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
 	defer cancelFunc()
 
+	if ctx.Value("dr-token") != nil {
+		c.c.SetToken(ctx.Value("dr-token").(string))
+	}
+
 	r := c.c.NewRequest(http.MethodGet, "/v1/sys/storage/raft/autopilot/configuration")
 
 	resp, err := c.c.rawRequestWithContext(ctx, r)

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1/doc.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:deepcopy-gen=package
+// +groupName=groupsnapshot.storage.k8s.io
+
+package v1alpha1

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1/register.go

@@ -0,0 +1,57 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+// GroupName is the group name use in this package.
+const GroupName = "groupsnapshot.storage.k8s.io"
+
+var (
+	// SchemeBuilder is the new scheme builder
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	// AddToScheme adds to scheme
+	AddToScheme = SchemeBuilder.AddToScheme
+	// SchemeGroupVersion is the group version used to register these objects.
+	SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
+)
+
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+func init() {
+	// We only register manually written functions here. The registration of the
+	// generated functions takes place in the generated files. The separation
+	// makes the code compile even when the generated files are missing.
+	SchemeBuilder.Register(addKnownTypes)
+}
+
+// addKnownTypes adds the set of types defined in this package to the supplied scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&VolumeGroupSnapshotClass{},
+		&VolumeGroupSnapshotClassList{},
+		&VolumeGroupSnapshot{},
+		&VolumeGroupSnapshotList{},
+		&VolumeGroupSnapshotContent{},
+		&VolumeGroupSnapshotContentList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1/types.go

@@ -0,0 +1,363 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+// +kubebuilder:object:generate=true
+package v1alpha1
+
+import (
+	core_v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
+)
+
+// VolumeGroupSnapshotSpec defines the desired state of a volume group snapshot.
+type VolumeGroupSnapshotSpec struct {
+	// Source specifies where a group snapshot will be created from.
+	// This field is immutable after creation.
+	// Required.
+	Source VolumeGroupSnapshotSource `json:"source" protobuf:"bytes,1,opt,name=source"`
+
+	// VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass
+	// requested by the VolumeGroupSnapshot.
+	// VolumeGroupSnapshotClassName may be left nil to indicate that the default
+	// class will be used.
+	// Empty string is not allowed for this field.
+	// +optional
+	VolumeGroupSnapshotClassName *string `json:"volumeGroupSnapshotClassName,omitempty" protobuf:"bytes,2,opt,name=volumeGroupSnapshotClassName"`
+}
+
+// VolumeGroupSnapshotSource specifies whether the underlying group snapshot should be
+// dynamically taken upon creation or if a pre-existing VolumeGroupSnapshotContent
+// object should be used.
+// Exactly one of its members must be set.
+// Members in VolumeGroupSnapshotSource are immutable.
+type VolumeGroupSnapshotSource struct {
+	// Selector is a label query over persistent volume claims that are to be
+	// grouped together for snapshotting.
+	// This labelSelector will be used to match the label added to a PVC.
+	// If the label is added or removed to a volume after a group snapshot
+	// is created, the existing group snapshots won't be modified.
+	// Once a VolumeGroupSnapshotContent is created and the sidecar starts to process
+	// it, the volume list will not change with retries.
+	// Required.
+	Selector metav1.LabelSelector `json:"selector" protobuf:"bytes,1,opt,name=selector"`
+
+	// VolumeGroupSnapshotContentName specifies the name of a pre-existing VolumeGroupSnapshotContent
+	// object representing an existing volume group snapshot.
+	// This field should be set if the volume group snapshot already exists and
+	// only needs a representation in Kubernetes.
+	// This field is immutable.
+	// +optional
+	VolumeGroupSnapshotContentName *string `json:"volumeGroupSnapshotContentName,omitempty" protobuf:"bytes,2,opt,name=volumeGroupSnapshotContentName"`
+}
+
+// VolumeGroupSnapshotStatus defines the observed state of volume group snapshot.
+type VolumeGroupSnapshotStatus struct {
+	// BoundVolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent
+	// object to which this VolumeGroupSnapshot object intends to bind to.
+	// If not specified, it indicates that the VolumeGroupSnapshot object has not
+	// been successfully bound to a VolumeGroupSnapshotContent object yet.
+	// NOTE: To avoid possible security issues, consumers must verify binding between
+	// VolumeGroupSnapshot and VolumeGroupSnapshotContent objects is successful
+	// (by validating that both VolumeGroupSnapshot and VolumeGroupSnapshotContent
+	// point at each other) before using this object.
+	// +optional
+	BoundVolumeGroupSnapshotContentName *string `json:"boundVolumeGroupSnapshotContentName,omitempty" protobuf:"bytes,1,opt,name=boundVolumeGroupSnapshotContentName"`
+
+	// CreationTime is the timestamp when the point-in-time group snapshot is taken
+	// by the underlying storage system.
+	// If not specified, it may indicate that the creation time of the group snapshot
+	// is unknown.
+	// The format of this field is a Unix nanoseconds time encoded as an int64.
+	// On Unix, the command date +%s%N returns the current time in nanoseconds
+	// since 1970-01-01 00:00:00 UTC.
+	// +optional
+	CreationTime *metav1.Time `json:"creationTime,omitempty" protobuf:"bytes,2,opt,name=creationTime"`
+
+	// ReadyToUse indicates if all the individual snapshots in the group are ready
+	// to be used to restore a group of volumes.
+	// ReadyToUse becomes true when ReadyToUse of all individual snapshots become true.
+	// If not specified, it means the readiness of a group snapshot is unknown.
+	// +optional
+	ReadyToUse *bool `json:"readyToUse,omitempty" protobuf:"varint,3,opt,name=readyToUse"`
+
+	// Error is the last observed error during group snapshot creation, if any.
+	// This field could be helpful to upper level controllers (i.e., application
+	// controller) to decide whether they should continue on waiting for the group
+	// snapshot to be created based on the type of error reported.
+	// The snapshot controller will keep retrying when an error occurs during the
+	// group snapshot creation. Upon success, this error field will be cleared.
+	// +optional
+	Error *snapshotv1.VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,4,opt,name=error,casttype=VolumeSnapshotError"`
+
+	// VolumeSnapshotRefList is the list of volume snapshot references for this
+	// group snapshot.
+	// The maximum number of allowed snapshots in the group is 100.
+	// +optional
+	VolumeSnapshotRefList []core_v1.ObjectReference `json:"volumeSnapshotRefList,omitempty" protobuf:"bytes,5,opt,name=volumeSnapshotRefList"`
+}
+
+//+genclient
+//+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// VolumeGroupSnapshot is a user's request for creating either a point-in-time
+// group snapshot or binding to a pre-existing group snapshot.
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Namespaced,shortName=vgs
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes."
+// +kubebuilder:printcolumn:name="VolumeGroupSnapshotClass",type=string,JSONPath=`.spec.volumeGroupSnapshotClassName`,description="The name of the VolumeGroupSnapshotClass requested by the VolumeGroupSnapshot."
+// +kubebuilder:printcolumn:name="VolumeGroupSnapshotContent",type=string,JSONPath=`.status.boundVolumeGroupSnapshotContentName`,description="Name of the VolumeGroupSnapshotContent object to which the VolumeGroupSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeGroupSnapshot and VolumeGroupSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object."
+// +kubebuilder:printcolumn:name="CreationTime",type=date,JSONPath=`.status.creationTime`,description="Timestamp when the point-in-time group snapshot was taken by the underlying storage system."
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+type VolumeGroupSnapshot struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines the desired characteristics of a group snapshot requested by a user.
+	// Required.
+	Spec VolumeGroupSnapshotSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
+	// Status represents the current information of a group snapshot.
+	// Consumers must verify binding between VolumeGroupSnapshot and
+	// VolumeGroupSnapshotContent objects is successful (by validating that both
+	// VolumeGroupSnapshot and VolumeGroupSnapshotContent point to each other) before
+	// using this object.
+	// +optional
+	Status *VolumeGroupSnapshotStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// VolumeGroupSnapshotList contains a list of VolumeGroupSnapshot objects.
+type VolumeGroupSnapshotList struct {
+	metav1.TypeMeta `json:",inline"`
+	// +optional
+	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+	// Items is the list of VolumeGroupSnapshots.
+	Items []VolumeGroupSnapshot `json:"items" protobuf:"bytes,2,rep,name=items"`
+}
+
+//+genclient
+//+genclient:nonNamespaced
+//+k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// VolumeGroupSnapshotClass specifies parameters that a underlying storage system
+// uses when creating a volume group snapshot. A specific VolumeGroupSnapshotClass
+// is used by specifying its name in a VolumeGroupSnapshot object.
+// VolumeGroupSnapshotClasses are non-namespaced.
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster,shortName=vgsclass;vgsclasses
+// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.driver`
+// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.deletionPolicy`,description="Determines whether a VolumeGroupSnapshotContent created through the VolumeGroupSnapshotClass should be deleted when its bound VolumeGroupSnapshot is deleted."
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+type VolumeGroupSnapshotClass struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard object's metadata.
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Driver is the name of the storage driver expected to handle this VolumeGroupSnapshotClass.
+	// Required.
+	Driver string `json:"driver" protobuf:"bytes,2,opt,name=driver"`
+
+	// Parameters is a key-value map with storage driver specific parameters for
+	// creating group snapshots.
+	// These values are opaque to Kubernetes and are passed directly to the driver.
+	// +optional
+	Parameters map[string]string `json:"parameters,omitempty" protobuf:"bytes,3,rep,name=parameters"`
+
+	// DeletionPolicy determines whether a VolumeGroupSnapshotContent created
+	// through the VolumeGroupSnapshotClass should be deleted when its bound
+	// VolumeGroupSnapshot is deleted.
+	// Supported values are "Retain" and "Delete".
+	// "Retain" means that the VolumeGroupSnapshotContent and its physical group
+	// snapshot on underlying storage system are kept.
+	// "Delete" means that the VolumeGroupSnapshotContent and its physical group
+	// snapshot on underlying storage system are deleted.
+	// Required.
+	DeletionPolicy snapshotv1.DeletionPolicy `json:"deletionPolicy" protobuf:"bytes,4,opt,name=deletionPolicy"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// VolumeGroupSnapshotClassList is a collection of VolumeGroupSnapshotClasses.
+// +kubebuilder:object:root=true
+type VolumeGroupSnapshotClassList struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Items is the list of VolumeGroupSnapshotClasses.
+	Items []VolumeGroupSnapshotClass `json:"items" protobuf:"bytes,2,rep,name=items"`
+}
+
+// +genclient
+// +genclient:nonNamespaced
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// VolumeGroupSnapshotContent represents the actual "on-disk" group snapshot object
+// in the underlying storage system
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster,shortName=vgsc;vgscs
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="ReadyToUse",type=boolean,JSONPath=`.status.readyToUse`,description="Indicates if all the individual snapshots in the group are ready to be used to restore a group of volumes."
+// +kubebuilder:printcolumn:name="DeletionPolicy",type=string,JSONPath=`.spec.deletionPolicy`,description="Determines whether this VolumeGroupSnapshotContent and its physical group snapshot on the underlying storage system should be deleted when its bound VolumeGroupSnapshot is deleted."
+// +kubebuilder:printcolumn:name="Driver",type=string,JSONPath=`.spec.driver`,description="Name of the CSI driver used to create the physical group snapshot on the underlying storage system."
+// +kubebuilder:printcolumn:name="VolumeGroupSnapshotClass",type=string,JSONPath=`.spec.volumeGroupSnapshotClassName`,description="Name of the VolumeGroupSnapshotClass from which this group snapshot was (or will be) created."
+// +kubebuilder:printcolumn:name="VolumeGroupSnapshotNamespace",type=string,JSONPath=`.spec.volumeGroupSnapshotRef.namespace`,description="Namespace of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent object is bound."
+// +kubebuilder:printcolumn:name="VolumeGroupSnapshot",type=string,JSONPath=`.spec.volumeGroupSnapshotRef.name`,description="Name of the VolumeGroupSnapshot object to which this VolumeGroupSnapshotContent object is bound."
+// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp`
+type VolumeGroupSnapshotContent struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Spec defines properties of a VolumeGroupSnapshotContent created by the underlying storage system.
+	// Required.
+	Spec VolumeGroupSnapshotContentSpec `json:"spec" protobuf:"bytes,2,opt,name=spec"`
+	// status represents the current information of a group snapshot.
+	// +optional
+	Status *VolumeGroupSnapshotContentStatus `json:"status,omitempty" protobuf:"bytes,3,opt,name=status"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// VolumeGroupSnapshotContentList is a list of VolumeGroupSnapshotContent objects
+// +kubebuilder:object:root=true
+type VolumeGroupSnapshotContentList struct {
+	metav1.TypeMeta `json:",inline"`
+	// Standard list metadata
+	// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
+	// +optional
+	metav1.ListMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Items is the list of VolumeGroupSnapshotContents.
+	Items []VolumeGroupSnapshotContent `json:"items" protobuf:"bytes,2,rep,name=items"`
+}
+
+// VolumeGroupSnapshotContentSpec describes the common attributes of a group snapshot content
+type VolumeGroupSnapshotContentSpec struct {
+	// VolumeGroupSnapshotRef specifies the VolumeGroupSnapshot object to which this
+	// VolumeGroupSnapshotContent object is bound.
+	// VolumeGroupSnapshot.Spec.VolumeGroupSnapshotContentName field must reference to
+	// this VolumeGroupSnapshotContent's name for the bidirectional binding to be valid.
+	// For a pre-existing VolumeGroupSnapshotContent object, name and namespace of the
+	// VolumeGroupSnapshot object MUST be provided for binding to happen.
+	// This field is immutable after creation.
+	// Required.
+	VolumeGroupSnapshotRef core_v1.ObjectReference `json:"volumeGroupSnapshotRef" protobuf:"bytes,1,opt,name=volumeGroupSnapshotRef"`
+
+	// DeletionPolicy determines whether this VolumeGroupSnapshotContent and the
+	// physical group snapshot on the underlying storage system should be deleted
+	// when the bound VolumeGroupSnapshot is deleted.
+	// Supported values are "Retain" and "Delete".
+	// "Retain" means that the VolumeGroupSnapshotContent and its physical group
+	// snapshot on underlying storage system are kept.
+	// "Delete" means that the VolumeGroupSnapshotContent and its physical group
+	// snapshot on underlying storage system are deleted.
+	// For dynamically provisioned group snapshots, this field will automatically
+	// be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field
+	// defined in the corresponding VolumeGroupSnapshotClass.
+	// For pre-existing snapshots, users MUST specify this field when creating the
+	// VolumeGroupSnapshotContent object.
+	// Required.
+	DeletionPolicy snapshotv1.DeletionPolicy `json:"deletionPolicy" protobuf:"bytes,2,opt,name=deletionPolicy"`
+
+	// Driver is the name of the CSI driver used to create the physical group snapshot on
+	// the underlying storage system.
+	// This MUST be the same as the name returned by the CSI GetPluginName() call for
+	// that driver.
+	// Required.
+	Driver string `json:"driver" protobuf:"bytes,3,opt,name=driver"`
+
+	// VolumeGroupSnapshotClassName is the name of the VolumeGroupSnapshotClass from
+	// which this group snapshot was (or will be) created.
+	// Note that after provisioning, the VolumeGroupSnapshotClass may be deleted or
+	// recreated with different set of values, and as such, should not be referenced
+	// post-snapshot creation.
+	// For dynamic provisioning, this field must be set.
+	// This field may be unset for pre-provisioned snapshots.
+	// +optional
+	VolumeGroupSnapshotClassName *string `json:"volumeGroupSnapshotClassName,omitempty" protobuf:"bytes,4,opt,name=volumeGroupSnapshotClassName"`
+
+	// Source specifies whether the snapshot is (or should be) dynamically provisioned
+	// or already exists, and just requires a Kubernetes object representation.
+	// This field is immutable after creation.
+	// Required.
+	Source VolumeGroupSnapshotContentSource `json:"source" protobuf:"bytes,5,opt,name=source"`
+}
+
+// VolumeGroupSnapshotContentStatus defines the observed state of VolumeGroupSnapshotContent.
+type VolumeGroupSnapshotContentStatus struct {
+	// VolumeGroupSnapshotHandle is a unique id returned by the CSI driver
+	// to identify the VolumeGroupSnapshot on the storage system.
+	// If a storage system does not provide such an id, the
+	// CSI driver can choose to return the VolumeGroupSnapshot name.
+	// +optional
+	VolumeGroupSnapshotHandle *string `json:"volumeGroupSnapshotHandle,omitempty" protobuf:"bytes,1,opt,name=volumeGroupSnapshotHandle"`
+
+	// CreationTime is the timestamp when the point-in-time group snapshot is taken
+	// by the underlying storage system.
+	// If not specified, it indicates the creation time is unknown.
+	// If not specified, it means the readiness of a group snapshot is unknown.
+	// The format of this field is a Unix nanoseconds time encoded as an int64.
+	// On Unix, the command date +%s%N returns the current time in nanoseconds
+	// since 1970-01-01 00:00:00 UTC.
+	// +optional
+	CreationTime *int64 `json:"creationTime,omitempty" protobuf:"varint,2,opt,name=creationTime"`
+
+	// ReadyToUse indicates if all the individual snapshots in the group are ready to be
+	// used to restore a group of volumes.
+	// ReadyToUse becomes true when ReadyToUse of all individual snapshots become true.
+	// +optional
+	ReadyToUse *bool `json:"readyToUse,omitempty" protobuf:"varint,3,opt,name=readyToUse"`
+
+	// Error is the last observed error during group snapshot creation, if any.
+	// Upon success after retry, this error field will be cleared.
+	// +optional
+	Error *snapshotv1.VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,4,opt,name=error,casttype=VolumeSnapshotError"`
+
+	// VolumeSnapshotContentRefList is the list of volume snapshot content references
+	// for this group snapshot.
+	// The maximum number of allowed snapshots in the group is 100.
+	// +optional
+	VolumeSnapshotContentRefList []core_v1.ObjectReference `json:"volumeSnapshotContentRefList,omitempty" protobuf:"bytes,5,opt,name=volumeSnapshotContentRefList"`
+}
+
+// VolumeGroupSnapshotContentSource represents the CSI source of a group snapshot.
+// Exactly one of its members must be set.
+// Members in VolumeGroupSnapshotContentSource are immutable.
+type VolumeGroupSnapshotContentSource struct {
+	// PersistentVolumeNames is a list of names of PersistentVolumes to be snapshotted
+	// together. It is specified for dynamic provisioning of the VolumeGroupSnapshot.
+	// This field is immutable.
+	// +optional
+	PersistentVolumeNames []string `json:"persistentVolumeNames,omitempty" protobuf:"bytes,1,opt,name=persistentVolumeNames"`
+
+	// VolumeGroupSnapshotHandle specifies the CSI "group_snapshot_id" of a pre-existing
+	// group snapshot on the underlying storage system for which a Kubernetes object
+	// representation was (or should be) created.
+	// This field is immutable.
+	// +optional
+	VolumeGroupSnapshotHandle *string `json:"volumeGroupSnapshotHandle,omitempty" protobuf:"bytes,2,opt,name=volumeGroupSnapshotHandle"`
+}

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumegroupsnapshot/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,398 @@
+//go:build !ignore_autogenerated
+// +build !ignore_autogenerated
+
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by deepcopy-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	v1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
+	corev1 "k8s.io/api/core/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshot) DeepCopyInto(out *VolumeGroupSnapshot) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	if in.Status != nil {
+		in, out := &in.Status, &out.Status
+		*out = new(VolumeGroupSnapshotStatus)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshot.
+func (in *VolumeGroupSnapshot) DeepCopy() *VolumeGroupSnapshot {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshot)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshot) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotClass) DeepCopyInto(out *VolumeGroupSnapshotClass) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	if in.Parameters != nil {
+		in, out := &in.Parameters, &out.Parameters
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotClass.
+func (in *VolumeGroupSnapshotClass) DeepCopy() *VolumeGroupSnapshotClass {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotClass)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshotClass) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotClassList) DeepCopyInto(out *VolumeGroupSnapshotClassList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]VolumeGroupSnapshotClass, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotClassList.
+func (in *VolumeGroupSnapshotClassList) DeepCopy() *VolumeGroupSnapshotClassList {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotClassList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshotClassList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotContent) DeepCopyInto(out *VolumeGroupSnapshotContent) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	if in.Status != nil {
+		in, out := &in.Status, &out.Status
+		*out = new(VolumeGroupSnapshotContentStatus)
+		(*in).DeepCopyInto(*out)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotContent.
+func (in *VolumeGroupSnapshotContent) DeepCopy() *VolumeGroupSnapshotContent {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotContent)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshotContent) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotContentList) DeepCopyInto(out *VolumeGroupSnapshotContentList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]VolumeGroupSnapshotContent, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotContentList.
+func (in *VolumeGroupSnapshotContentList) DeepCopy() *VolumeGroupSnapshotContentList {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotContentList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshotContentList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotContentSource) DeepCopyInto(out *VolumeGroupSnapshotContentSource) {
+	*out = *in
+	if in.PersistentVolumeNames != nil {
+		in, out := &in.PersistentVolumeNames, &out.PersistentVolumeNames
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.VolumeGroupSnapshotHandle != nil {
+		in, out := &in.VolumeGroupSnapshotHandle, &out.VolumeGroupSnapshotHandle
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotContentSource.
+func (in *VolumeGroupSnapshotContentSource) DeepCopy() *VolumeGroupSnapshotContentSource {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotContentSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotContentSpec) DeepCopyInto(out *VolumeGroupSnapshotContentSpec) {
+	*out = *in
+	out.VolumeGroupSnapshotRef = in.VolumeGroupSnapshotRef
+	if in.VolumeGroupSnapshotClassName != nil {
+		in, out := &in.VolumeGroupSnapshotClassName, &out.VolumeGroupSnapshotClassName
+		*out = new(string)
+		**out = **in
+	}
+	in.Source.DeepCopyInto(&out.Source)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotContentSpec.
+func (in *VolumeGroupSnapshotContentSpec) DeepCopy() *VolumeGroupSnapshotContentSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotContentSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotContentStatus) DeepCopyInto(out *VolumeGroupSnapshotContentStatus) {
+	*out = *in
+	if in.VolumeGroupSnapshotHandle != nil {
+		in, out := &in.VolumeGroupSnapshotHandle, &out.VolumeGroupSnapshotHandle
+		*out = new(string)
+		**out = **in
+	}
+	if in.CreationTime != nil {
+		in, out := &in.CreationTime, &out.CreationTime
+		*out = new(int64)
+		**out = **in
+	}
+	if in.ReadyToUse != nil {
+		in, out := &in.ReadyToUse, &out.ReadyToUse
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Error != nil {
+		in, out := &in.Error, &out.Error
+		*out = new(v1.VolumeSnapshotError)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.VolumeSnapshotContentRefList != nil {
+		in, out := &in.VolumeSnapshotContentRefList, &out.VolumeSnapshotContentRefList
+		*out = make([]corev1.ObjectReference, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotContentStatus.
+func (in *VolumeGroupSnapshotContentStatus) DeepCopy() *VolumeGroupSnapshotContentStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotContentStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotList) DeepCopyInto(out *VolumeGroupSnapshotList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]VolumeGroupSnapshot, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotList.
+func (in *VolumeGroupSnapshotList) DeepCopy() *VolumeGroupSnapshotList {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VolumeGroupSnapshotList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotSource) DeepCopyInto(out *VolumeGroupSnapshotSource) {
+	*out = *in
+	in.Selector.DeepCopyInto(&out.Selector)
+	if in.VolumeGroupSnapshotContentName != nil {
+		in, out := &in.VolumeGroupSnapshotContentName, &out.VolumeGroupSnapshotContentName
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotSource.
+func (in *VolumeGroupSnapshotSource) DeepCopy() *VolumeGroupSnapshotSource {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotSpec) DeepCopyInto(out *VolumeGroupSnapshotSpec) {
+	*out = *in
+	in.Source.DeepCopyInto(&out.Source)
+	if in.VolumeGroupSnapshotClassName != nil {
+		in, out := &in.VolumeGroupSnapshotClassName, &out.VolumeGroupSnapshotClassName
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotSpec.
+func (in *VolumeGroupSnapshotSpec) DeepCopy() *VolumeGroupSnapshotSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeGroupSnapshotStatus) DeepCopyInto(out *VolumeGroupSnapshotStatus) {
+	*out = *in
+	if in.BoundVolumeGroupSnapshotContentName != nil {
+		in, out := &in.BoundVolumeGroupSnapshotContentName, &out.BoundVolumeGroupSnapshotContentName
+		*out = new(string)
+		**out = **in
+	}
+	if in.CreationTime != nil {
+		in, out := &in.CreationTime, &out.CreationTime
+		*out = (*in).DeepCopy()
+	}
+	if in.ReadyToUse != nil {
+		in, out := &in.ReadyToUse, &out.ReadyToUse
+		*out = new(bool)
+		**out = **in
+	}
+	if in.Error != nil {
+		in, out := &in.Error, &out.Error
+		*out = new(v1.VolumeSnapshotError)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.VolumeSnapshotRefList != nil {
+		in, out := &in.VolumeSnapshotRefList, &out.VolumeSnapshotRefList
+		*out = make([]corev1.ObjectReference, len(*in))
+		copy(*out, *in)
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeGroupSnapshotStatus.
+func (in *VolumeGroupSnapshotStatus) DeepCopy() *VolumeGroupSnapshotStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeGroupSnapshotStatus)
+	in.DeepCopyInto(out)
+	return out
+}

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/types.go

@@ -183,6 +183,11 @@ type VolumeSnapshotStatus struct {
 	// snapshot creation. Upon success, this error field will be cleared.
 	// +optional
 	Error *VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error,casttype=VolumeSnapshotError"`
+
+	// VolumeGroupSnapshotName is the name of the VolumeGroupSnapshot of which this
+	// VolumeSnapshot is a part of.
+	// +optional
+	VolumeGroupSnapshotName *string `json:"volumeGroupSnapshotName,omitempty" protobuf:"bytes,6,opt,name=volumeGroupSnapshotName"`
 }
 
 // +genclient
@@ -415,6 +420,11 @@ type VolumeSnapshotContentStatus struct {
 	// Upon success after retry, this error field will be cleared.
 	// +optional
 	Error *VolumeSnapshotError `json:"error,omitempty" protobuf:"bytes,5,opt,name=error,casttype=VolumeSnapshotError"`
+
+	// VolumeGroupSnapshotContentName is the name of the VolumeGroupSnapshotContent of
+	// which this VolumeSnapshotContent is a part of.
+	// +optional
+	VolumeGroupSnapshotContentName *string `json:"volumeGroupSnapshotContentName,omitempty" protobuf:"bytes,6,opt,name=volumeGroupSnapshotContentName"`
 }
 
 // DeletionPolicy describes a policy for end-of-life maintenance of volume snapshot contents

vendor/github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1/zz_generated.deepcopy.go

@@ -2,7 +2,7 @@
 // +build !ignore_autogenerated
 
 /*
-Copyright 2022 The Kubernetes Authors.
+Copyright 2023 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -271,6 +271,11 @@ func (in *VolumeSnapshotContentStatus) DeepCopyInto(out *VolumeSnapshotContentSt
 		*out = new(VolumeSnapshotError)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.VolumeGroupSnapshotContentName != nil {
+		in, out := &in.VolumeGroupSnapshotContentName, &out.VolumeGroupSnapshotContentName
+		*out = new(string)
+		**out = **in
+	}
 	return
 }
 
@@ -417,6 +422,11 @@ func (in *VolumeSnapshotStatus) DeepCopyInto(out *VolumeSnapshotStatus) {
 		*out = new(VolumeSnapshotError)
 		(*in).DeepCopyInto(*out)
 	}
+	if in.VolumeGroupSnapshotName != nil {
+		in, out := &in.VolumeGroupSnapshotName, &out.VolumeGroupSnapshotName
+		*out = new(string)
+		**out = **in
+	}
 	return
 }