Merge pull request #96 from ceph/devel

Sync upstream devel to downstream devel

.github/dependabot.yml

@@ -49,3 +49,13 @@ updates:
       - ci/skip/e2e
     commit-message:
       prefix: "rebase"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: weekly
+    rebase-strategy: disabled
+    labels:
+      - rebase
+      - ci/skip/e2e
+    commit-message:
+      prefix: rebase

.github/workflows/build-multi-stage.yaml

@@ -5,14 +5,23 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   codespell:
     name: multi-arch-build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: multi-arch-build
+        # yamllint disable-line rule:line-length
+        if: ${{ ! contains(github.event.pull_request.labels.*.name, 'ci/skip/multi-arch-build') }}
         # podman cannot pull images with both tag and digest
         # https://github.com/containers/buildah/issues/1407
         # use docker to build images
         run: CONTAINER_CMD=docker ./scripts/build-multi-arch-image.sh
+      - name: single-arch-build
+        # yamllint disable-line rule:line-length
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'ci/skip/multi-arch-build') }}
+        run: make containerized-build

.github/workflows/codespell.yaml

@@ -7,11 +7,14 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   codespell:
     name: codespell
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: codespell
         run: CONTAINER_CMD=docker make containerized-test TARGET=codespell

.github/workflows/commitlint.yaml

@@ -5,13 +5,16 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   commitlint:
     name: commitlint
     if: ${{ github.actor != 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: commitlint

.github/workflows/dependency-review.yaml

@@ -0,0 +1,20 @@
+---
+name: 'Dependency Review'
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - '*'
+
+
+permissions:
+  contents: read
+
+jobs:
+  dependency-review:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Checkout Repository'
+        uses: actions/checkout@v3
+      - name: 'Dependency Review'
+        uses: actions/dependency-review-action@v1

.github/workflows/go-test.yaml

@@ -5,18 +5,21 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   go-test:
     name: go-test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: go-test
         run: CONTAINER_CMD=docker make containerized-test TARGET=go-test
   go-test-api:
     name: go-test-api
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: go-test-api
         run: CONTAINER_CMD=docker make containerized-test TARGET=go-test-api

.github/workflows/golangci-lint.yaml

@@ -5,11 +5,14 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   golangci-lint:
     name: golangci-lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: golangci-lint
         run: CONTAINER_CMD=docker make containerized-test TARGET=go-lint

.github/workflows/lint-extras.yaml

@@ -5,11 +5,14 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   lint-extras:
     name: lint-extras
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: lint-extras
         run: CONTAINER_CMD=docker make containerized-test TARGET=lint-extras

.github/workflows/mod-check.yaml

@@ -5,11 +5,14 @@ on:
   pull_request:
     branches:
       - '*'
+permissions:
+  contents: read
+
 jobs:
   mod-check:
     name: mod-check
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: mod-check
         run: CONTAINER_CMD=docker make containerized-test TARGET=mod-check

.github/workflows/publish-artifacts.yaml

@@ -9,16 +9,19 @@ on:
       - devel
       # Push events to branches matching refs/heads/release-v*
       - 'release-v*'
+permissions:
+  contents: read
+
 jobs:
   push:
     name: Publish artifacts
     runs-on: ubuntu-latest
     if: github.repository == 'ceph/ceph-csi'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Login to Quay
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           registry: quay.io
           username: ${{ secrets.QUAY_IO_USERNAME }}

.github/workflows/retest.yaml

@@ -5,6 +5,10 @@ on:
   schedule:
     # Run the retest action every 30 minutes
     - cron: "30 * * * *"
+
+permissions:
+  contents: read
+
 jobs:
   retest:
     runs-on: ubuntu-latest

.github/workflows/stale.yaml

@@ -7,12 +7,18 @@ on:
     # Run the stalebot every day at 9pm UTC
     - cron: "00 21 * * *"
 # yamllint disable rule:line-length
+permissions:
+  contents: read
+
 jobs:
   stale:
+    permissions:
+      issues: write  # for actions/stale to close stale issues
+      pull-requests: write  # for actions/stale to close stale PRs
     runs-on: ubuntu-18.04
     if: github.repository == 'ceph/ceph-csi'
     steps:
-      - uses: actions/stale@v3
+      - uses: actions/stale@v5
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           days-before-issue-stale: 30

.github/workflows/test-retest-action.yaml

@@ -5,11 +5,14 @@ on:
   pull_request:
     branches: [devel]
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Docker build
         # Run cd to avoid loading complete cephcsi directory in docker context

README.md

@@ -113,6 +113,10 @@ for its support details.
 |        | Dynamically provision, de-provision File mode RWX volume  | Alpha          | >= v3.6.0          | >= v1.0.0        | Pacific (>=16.2.0)   | >= v1.14.0         |
 |        | Dynamically provision, de-provision File mode ROX volume  | Alpha          | >= v3.6.0          | >= v1.0.0        | Pacific (>=16.2.0)   | >= v1.14.0         |
 |        | Dynamically provision, de-provision File mode RWOP volume | Alpha          | >= v3.6.0          | >= v1.5.0        | Pacific (>=16.2.0)   | >= v1.22.0         |
+|        | Expand volume                                             | Alpha          | >= v3.7.0          | >= v1.1.0        | Pacific (>=16.2.0)   | >= v1.15.0         |
+|        | Creating and deleting snapshot                            | Alpha          | >= v3.7.0          | >= v1.1.0        | Pacific (>=16.2.0)   | >= v1.17.0         |
+|        | Provision volume from snapshot                            | Alpha          | >= v3.7.0          | >= v1.1.0        | Pacific (>=16.2.0)   | >= v1.17.0         |
+|        | Provision volume from another volume                      | Alpha          | >= v3.7.0          | >= v1.1.0        | Pacific (>=16.2.0)   | >= v1.16.0         |
 
 `NOTE`: The `Alpha` status reflects possible non-backward
 compatible changes in the future, and is thus not recommended

build.env

@@ -52,11 +52,11 @@ CSI_ATTACHER_VERSION=v3.4.0
 CSI_SNAPSHOTTER_VERSION=v5.0.1
 CSI_PROVISIONER_VERSION=v3.1.0
 CSI_RESIZER_VERSION=v1.4.0
-CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.4.0
+CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.5.1
 
 # e2e settings
 # - enable CEPH_CSI_RUN_ALL_TESTS when running tests with if it has root
 #   permissions on the host
 #CEPH_CSI_RUN_ALL_TESTS=true
-E2E_TIMEOUT=120m
+E2E_TIMEOUT=150m
 DEPLOY_TIMEOUT=10

charts/ceph-csi-cephfs/README.md

@@ -125,8 +125,6 @@ charts and their default values.
 | `provisioner.tolerations`                      | Specifies the tolerations for provisioner deployment                                                                                                 | `{}`                                               |
 | `provisioner.affinity`                         | Specifies the affinity for provisioner deployment                                                                                                    | `{}`                                               |
 | `provisioner.podSecurityPolicy.enabled`        | Specifies whether podSecurityPolicy is enabled                                                                                                       | `false`                                            |
-| `topology.enabled`                             | Specifies whether topology based provisioning support should be exposed by CSI                                                                       | `false`                                            |
-| `topology.domainLabels`                        | DomainLabels define which node labels to use as domains for CSI nodeplugins to advertise their domains                                               | `{}`                                               |
 | `provisionerSocketFile`                        | The filename of the provisioner socket                                                                                                               | `csi-provisioner.sock`                             |
 | `pluginSocketFile`                             | The filename of the plugin socket                                                                                                                    | `csi.sock`                                         |
 | `kubeletDir`                                   | Kubelet working directory                                                                                                                            | `/var/lib/kubelet`                                 |

charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml

@@ -75,9 +75,6 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v={{ .Values.logLevel }}"
             - "--drivername=$(DRIVER_NAME)"
-{{- if .Values.topology.enabled }}
-            - "--domainlabels={{ .Values.topology.domainLabels | join "," }}"
-{{- end }}
 {{- if .Values.nodeplugin.profiling.enabled }}
             - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
 {{- end }}

charts/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml

@@ -53,12 +53,4 @@ rules:
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
 {{- end -}}
-{{- if .Values.topology.enabled }}
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["csinodes"]
-    verbs: ["get", "list", "watch"]
-{{- end }}
 {{- end -}}

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -62,9 +62,6 @@ spec:
             - "--leader-election=true"
             - "--retry-interval-start=500ms"
             - "--extra-create-metadata=true"
-{{- if .Values.topology.enabled }}
-            - "--feature-gates=Topology=true"
-{{- end }}
           env:
             - name: ADDRESS
               value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
@@ -118,6 +115,7 @@ spec:
             - "--leader-election"
             - "--retry-interval-start=500ms"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=RecoverVolumeExpansionFailure=true"
           env:
             - name: ADDRESS
               value: "unix:///csi/{{ .Values.provisionerSocketFile }}"

charts/ceph-csi-cephfs/values.yaml

@@ -81,7 +81,7 @@ nodeplugin:
   registrar:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
-      tag: v2.4.0
+      tag: v2.5.1
       pullPolicy: IfNotPresent
     resources: {}
 
@@ -206,18 +206,6 @@ provisioner:
 # selinux-enabled filesystems
 selinuxMount: true
 
-topology:
-  # Specifies whether topology based provisioning support should
-  # be exposed by CSI
-  enabled: false
-  # domainLabels define which node labels to use as domains
-  # for CSI nodeplugins to advertise their domains
-  # NOTE: the value here serves as an example and needs to be
-  # updated with node labels that define domains of interest
-  domainLabels:
-    - failure-domain/region
-    - failure-domain/zone
-
 storageClass:
   # Specifies whether the Storage class should be created
   create: false

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -85,6 +85,7 @@ spec:
             - "--leader-election"
             - "--retry-interval-start=500ms"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=RecoverVolumeExpansionFailure=true"
           env:
             - name: ADDRESS
               value: "unix:///csi/{{ .Values.provisionerSocketFile }}"

charts/ceph-csi-rbd/values.yaml

@@ -104,7 +104,7 @@ nodeplugin:
   registrar:
     image:
       repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
-      tag: v2.4.0
+      tag: v2.5.1
       pullPolicy: IfNotPresent
     resources: {}
 
@@ -199,8 +199,12 @@ provisioner:
 
   provisioner:
     image:
-      repository: registry.k8s.io/sig-storage/csi-provisioner
-      tag: v3.1.0
+      # TODO: replace with released image version.
+      # canary image is being to be used to test pvc-pvc clone
+      # with differe sc feature.
+      # see: https://github.com/kubernetes-csi/external-provisioner/pull/699
+      repository: gcr.io/k8s-staging-sig-storage/csi-provisioner
+      tag: canary
       pullPolicy: IfNotPresent
     resources: {}
 

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -68,6 +68,7 @@ spec:
             - "--leader-election"
             - "--retry-interval-start=500ms"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=RecoverVolumeExpansionFailure=true"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock

deploy/cephfs/kubernetes/csi-cephfsplugin.yaml

@@ -27,7 +27,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"

deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml

@@ -10,9 +10,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: cephfs-external-provisioner-runner
 rules:
-  - apiGroups: [""]
-    resources: ["nodes"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["get", "list"]
@@ -36,7 +33,7 @@ rules:
     verbs: ["get", "list", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
@@ -77,10 +74,6 @@ metadata:
   namespace: default
   name: cephfs-external-provisioner-cfg
 rules:
-  # remove this once we stop supporting v1.0.0
-  - apiGroups: [""]
-    resources: ["configmaps"]
-    verbs: ["get", "list", "create", "delete"]
   - apiGroups: ["coordination.k8s.io"]
     resources: ["leases"]
     verbs: ["get", "watch", "list", "delete", "update", "create"]

deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml

@@ -54,6 +54,36 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+        - name: csi-resizer
+          image: registry.k8s.io/sig-storage/csi-resizer:v1.4.0
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--leader-election"
+            - "--retry-interval-start=500ms"
+            - "--handle-volume-inuse-error=false"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-snapshotter
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--leader-election=true"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
         - name: csi-nfsplugin
           # for stable functionality replace canary with latest release version
           image: quay.io/cephcsi/cephcsi:canary

deploy/nfs/kubernetes/csi-nfsplugin.yaml

@@ -44,7 +44,7 @@ spec:
                 fieldRef:
                   apiVersion: v1
                   fieldPath: spec.nodeName
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
           imagePullPolicy: IfNotPresent
           livenessProbe:
             exec:
@@ -86,7 +86,7 @@ spec:
                   fieldPath: spec.nodeName
             - name: CSI_ENDPOINT
               value: unix:///csi/csi.sock
-          image: registry.k8s.io/sig-storage/nfsplugin:v3.1.0
+          image: registry.k8s.io/sig-storage/nfsplugin:v4.0.0
           imagePullPolicy: IfNotPresent
           livenessProbe:
             failureThreshold: 5

deploy/nfs/kubernetes/csi-nodeplugin-psp.yaml

@@ -43,7 +43,9 @@ spec:
       readOnly: false
     - pathPrefix: '/var/lib/kubelet/plugins'
       readOnly: false
-
+  hostPorts:
+    - min: 29653
+      max: 29653
 ---
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1

deploy/nfs/kubernetes/csi-provisioner-rbac.yaml

@@ -39,6 +39,18 @@ rules:
   - apiGroups: ["storage.k8s.io"]
     resources: ["csinodes"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -47,7 +47,11 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v3.1.0
+          # TODO: replace with released image version.
+          # Canary image is being to be used to test pvc-pvc clone
+          # with differe sc feature.
+          # see: https://github.com/kubernetes-csi/external-provisioner/pull/699
+          image: gcr.io/k8s-staging-sig-storage/csi-provisioner:canary
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
@@ -104,6 +108,7 @@ spec:
             - "--leader-election"
             - "--retry-interval-start=500ms"
             - "--handle-volume-inuse-error=false"
+            - "--feature-gates=RecoverVolumeExpansionFailure=true"
           env:
             - name: ADDRESS
               value: unix:///csi/csi-provisioner.sock

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -29,7 +29,7 @@ spec:
           securityContext:
             privileged: true
             allowPrivilegeEscalation: true
-          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.4.0
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1
           args:
             - "--v=5"
             - "--csi-address=/csi/csi.sock"

docs/design/proposals/cephfs-fscrypt.md

@@ -0,0 +1,284 @@
+# Ceph Filesystem fscrypt Support
+
+## Problem Description
+
+As a Ceph Container Storage Interface (Ceph CSI) user, I want a cloud native way
+to manage keys and enable encryption on Ceph Filesystem (CephFS) volumes.
+
+In order to access encrypted volumes without Ceph CSI, this can be done by unlocking
+volumes with user space tools.
+
+## Background
+
+*fscrypt* or *FSCrypt* is a Linux Kernel feature that allows the filesystem
+to support the transparent encryption of files and directories. Local
+filesystems like ext4 and F2FS (Flash-Friendly File System) support this feature
+already.
+
+Work is in progress to add fscrypt support to CephFS for filesystem-level encryption.
+
+- [FSCrypt Kernel Documentation](https://www.kernel.org/doc/html/latest/filesystems/fscrypt.html)
+- Management Tools
+  - [`fscrypt`](https://github.com/google/fscrypt)
+  - [`fscryptctl`](https://github.com/google/fscryptctl)
+- [Ceph Feature Tracker: "Add fscrypt support to the kernel CephFS client"](https://tracker.ceph.com/issues/46690)
+- [`fscrypt` design document](https://goo.gl/55cCrI)
+
+**NOTE:** In this document, fscrypt refers to the filesystem-level encryption
+feature, while `fscrypt` specifically refers to the user space tool.
+
+## Terminology
+
+- *FSCrypt*, *fscrypt* - Linux Kernel filesystem-level encryption feature
+- `fscrypt` (code formatted) - User space tool manage keys and encryption policies
+- `fscryptctl` (code formatted) - Low-level user space tool manage
+  keys and encryption policies
+- *subvolume* - CephFS subvolume
+- *unlocking* - Using a key to make an encrypted filesystem
+  accessible in plain text
+- *protector* (fscrypt) - A single method or secret plus data used to
+  derive a *protector key*. Example: user login passphrase
+- *protector key* (fscrypt) - A symmetric key derived from an external
+  source. Used by a *policy* to unwrap a *policy key*
+- *policy key* (fscrypt) - An encryption key passed to the kernel to
+  unlock a directory
+- *policy* (fscrypt) - A collection of directories protected and unlocked as a unit
+- *KMS* - Key management system
+
+## User Visible Change
+
+Similar to the existing RADOS Block Device (RBD) encryption support,
+we propose adding encryption support in the configuration and Key
+Management Service (KMS) integration.
+
+In this example, a user may enable encryption using storage class keys similar
+to RBD. Ceph CSI then configures and unlocks the persistent volumes and CephFS
+subvolumes.
+
+Due to the way `fscrypt` stores metadata, subvolumes have a regular root
+directory containing  a `/.fscrypt` directory and a
+`/ceph-csi-encrypted` directory. The first contains `fscrypt` metadata; the
+latter is the fscrypt-enabled directory made that is accessible to pods.
+
+Example configuration using a secrets-based KMS:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cephfs-storage-encryption-secret
+stringData:
+  encryptionPassphrase: verysecretpassword
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-cephfs-sc-encrypted
+provisioner: cephfs.csi.ceph.com
+parameters:
+  clusterID: <cluster-id>
+  fsName: cephfs
+
+  encrypted: "true"
+  encryptionKMSID: "user-ns-secrets-metadata"
+
+  csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: default
+  csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
+  csi.storage.k8s.io/controller-expand-secret-namespace: default
+  csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
+  csi.storage.k8s.io/node-stage-secret-namespace: default
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+mountOptions:
+  - debug
+```
+
+The change will leverage the existing Ceph CSI KMS and support any
+integration now available to RBD encryption
+
+## Implementation
+
+We suggest to leverage the encryption features in Ceph CSI and integrate
+that with `fscrypt`, a Go tool, and the library for key management and configuration
+of the fscrypt kernel feature.
+
+Ceph CSI and `fscrypt` have a lot of overlap between their key management
+features. The Key Management section will go into detail on how and where keys
+are managed.
+
+- Ceph CSI provides the user facing configuration and access to key
+  management systems
+- `fscrypt` handles key derivation, storage of wrapped keys and metadata
+
+The current CephFS subvolume root will remain untouched with the exception that
+the subvolume root is not bind mounted into the pod, but rather a well-known
+subdirectory. The root will contain a `/.fscrypt` directory managed by `fscrypt`.
+
+`fscrypt` requires access to a mounted filesystem and therefore the encryption setup
+must take place in the `NodeStageVolume` request handler instead of `CreateVolume`.
+This is the same case for RBD. The set up will take place right between
+subvolume mount and bind mount to the container namespace.
+
+Additional checks after unlocking will ensure that a container operates on an
+unlocked encrypted directory and never on directory that has fscrypt enabled.
+
+### Key Management
+
+```mermaid
+graph LR
+    vault[Vault, Default] -->|data encryption key| csi_kms_integrated
+    secrets[Passphrase<br>K8s secrets, AWS, IBM Key Protect] -->|passphrase| csi_kms_metadata
+
+
+    subgraph ceph_csi_kms[Ceph CSI KMS]
+        csi_kms_metadata['metadata' type DEK store]
+        csi_kms_integrated['integrated' type DEK store]
+
+    end
+
+
+    subgraph protector[fscrypt protector]
+        custom_passphrase[CustomPasswordSource];
+        raw_passphrase[RawKeySource];
+
+        protector_metadata[(/.fscrypt/protectors)];
+
+        protector_metadata -->|wrapped key| custom_passphrase;
+        protector_metadata -->|wrapped key| raw_passphrase;
+
+    end
+
+    subgraph policy[fscrypt policy]
+        policy_unwrap
+        policy_metadata[(/.fscrypt/policies)]
+
+        policy_metadata -->|wrapped key| policy_unwrap
+    end
+
+
+    csi_kms_metadata -->|DEK| custom_passphrase
+    csi_kms_integrated -->|DEK| raw_passphrase
+
+    custom_passphrase -->|protector key| policy_unwrap
+    raw_passphrase -->|protector key| policy_unwrap
+
+    policy_unwrap -->|policy key| kernel[Kernel API]
+
+```
+
+The diagram shows the keys flowing from Ceph CSI to the Kernel API unlocking
+a directory. On the way, key material from Ceph CSI passes two key derivation
+steps in fscrypt:
+
+- *protectors* and
+- *policies*
+
+`fscrypt` supports multiple *protectors*. These may source secrets from login
+passwords, custom passwords or soon Ceph CSI. Unlocking a protector
+yields a *protector key* that is then used to unlock a *policy*.
+
+A *policy* may unlock multiple directories. In our case there will be
+only a single policy for a single well-known directory on the subvolume root.
+A policy is used to derive a *policy key*, which is passed to the Kernel API
+along with other settings, such as the desired encryption algorithm.
+
+Going back to the beginning of the diagram and looking at the interface between
+`fscrypt` and Ceph CSI one can see that the two data encryption key (DEK)
+styles (*metadata* and *integrated*) map to different `fscrypt` protectors.
+
+The `fscrypt` protector of key sources `CustomPasswordSource` and `RawKeySource`
+differ in how they derive a key from a source. Refer to the `fscrypt` design
+doc for details.
+
+Metadata DEKs: In the RBD case, Ceph CSI stores a wrapped key in the
+RBD volume metadata and then a user configured secret (for example, a Kubernetes
+secret) is passed to a key derivation function (KDF) to then unwrap the
+key. The resulting key unlocks the volume.
+
+Since `fscrypt` already stores wrapped keys there is no need for an
+extra layer of wrapping. We can also skip the KDF and use a
+`CustomPasswordSource` to pass the Ceph CSI secret directly to
+`fscrypt`.
+
+With integrated DEKs (for example, Vault) Ceph CSI uses a key from a KMS
+directly. To integrate this with `fscrypt` we use a
+`RawKeySource`, that is similar to a `CustomPasswordSource`, but skips
+the KDF.
+
+As the diagram shows, both policies and protectors require a metadata
+store. The default `fscrypt` data store is in a `/.fscrypt`
+directory under a filesystem root. The `fscrypt` design doc details
+alternatives and explains what data is stored.
+
+To be compatible with `fscrypt`, this directory requires support as
+well. The downside of this is that we lose the CephFS subvolume root to
+metadata and encrypted data will reside under a well-known
+subdirectory (for example, `/ceph-csi-encrypted`).
+
+## Dependencies
+
+The proposed change is tailored to CephFS and requires CephFS support
+to work *with* CephFS. The kernel APIs however are not specific to
+CephFS and are unlikely to change as they only deal with configuration
+and key management. There is no direct dependency on CephFS. Using the
+proposed features will simply fail at runtime, when neither Ceph nor
+the Kernel have the appropriate support. At build time this feature
+does not require CephFS fscrypt support.
+
+Runtime dependencies:
+
+- Kernel >= v5.4 with `CONFIG_FS_ENCRYPTION=y`
+- CephFS kernel client fscrypt support [Ceph Feature Tracker](https://tracker.ceph.com/issues/46690)
+
+Build dependencies:
+
+- `google/fscrypt` library, which has minimal build dependencies
+  ([fscrypt doc](https://github.com/google/fscrypt#building-and-installing))
+
+## Alternatives
+
+### [ceph-csi-kms] Key Management: Policy Key Directly From Ceph CSI
+
+A simpler approach to the one proposed above, but incompatible with
+`fscrypt`. To unlock a subvolume, the user would have to use Ceph CSI.
+The implementation is similar to the RBD encryption feature.
+It uses the low-level `fscryptctl` tool to set a policy key
+from a Ceph CSI data encryption key.
+
+Ceph CSI KMS requires metadata data encryption key storage that can use
+xattrs on a mounted CephFS filesystem.
+
+A prototype showing this approach is available: [repository](https://github.com/irq0/ceph-csi/tree/wip/fscrypt)
+
+Benefits:
+
+- Simpler key wrapping
+- No `/.fscrypt` on the subvolume root
+
+Drawbacks:
+
+- `fscryptctl` is a C tool and does not lend itself to be integrated into Ceph CSI
+- Incompatible with `fscrypt`
+- Does not support unlocking with any of possibly multiple keys
+  configured (`fscrypt` protectors feature)
+
+### [manual] Manual Setup
+
+For completeness, a user may set up FSCrypt without any support in
+Ceph CSI. Both `fscrypt` and `fscryptctl` work in containers
+and may even be used with the proposed change or alternative [ceph-csi-kms].
+
+The following links provide examples from the documentation that also apply to
+CephFS:
+
+- [fscrypt example usage](https://github.com/google/fscrypt#example-usage)
+- [fscryptctl example usage](https://github.com/google/fscryptctl#example-usage)
+
+### [subdirs] Support Unlocking Arbitrary subdirs (instead of subvolume basis)
+
+An extension to the proposal: As mentioned in the implementation section, a
+`fscrypt` policy may apply to multiple directories and from a set of protectors
+any suffices to unlock a policy. A user may configure a complex mapping of
+subdirectories and Ceph CSI secret sources to unlock different parts of a CephFS
+subvolume with different keys.

docs/development-guide.md

@@ -251,6 +251,7 @@ A few labels interact with automation around the pull requests:
 * ready-to-merge: This PR is ready to be merged and it doesn't need second review
 * DNM: DO NOT MERGE (Mergify will not merge this PR)
 * ci/skip/e2e: skip running e2e CI jobs
+* ci/skip/multi-arch-build: skip building container images for different architectures
 
 **Review Process:**
 Once your PR has been submitted for review the following criteria will

e2e/README.md

@@ -104,6 +104,7 @@ are available while running tests:
 | is-openshift      | Run in OpenShift compatibility mode, skips certain new feature tests                              |
 | filesystem        | Name of the CephFS filesystem (default: "myfs")                                                   |
 | clusterid         | Use the Ceph cluster id in the StorageClasses and SnapshotClasses (default: `ceph fsid` detected) |
+| nfs-driver        | Name of the driver to use for provisioning NFS-volumes (default: "nfs.csi.ceph.com")              |
 
 ## E2E for snapshot
 

e2e/cephfs.go

@@ -69,6 +69,7 @@ func deleteCephfsPlugin() {
 
 func createORDeleteCephfsResources(action kubectlAction) {
 	resources := []ResourceDeployer{
+		// shared resources
 		&yamlResource{
 			filename:     cephFSDirPath + csiDriverObject,
 			allowMissing: true,
@@ -77,11 +78,7 @@ func createORDeleteCephfsResources(action kubectlAction) {
 			filename:     examplePath + cephConfconfigMap,
 			allowMissing: true,
 		},
-		&yamlResourceNamespaced{
-			filename:   cephFSDirPath + cephFSProvisioner,
-			namespace:  cephCSINamespace,
-			oneReplica: true,
-		},
+		// dependencies for provisioner
 		&yamlResourceNamespaced{
 			filename:  cephFSDirPath + cephFSProvisionerRBAC,
 			namespace: cephCSINamespace,
@@ -90,10 +87,13 @@ func createORDeleteCephfsResources(action kubectlAction) {
 			filename:  cephFSDirPath + cephFSProvisionerPSP,
 			namespace: cephCSINamespace,
 		},
+		// the provisioner itself
 		&yamlResourceNamespaced{
-			filename:  cephFSDirPath + cephFSNodePlugin,
-			namespace: cephCSINamespace,
+			filename:   cephFSDirPath + cephFSProvisioner,
+			namespace:  cephCSINamespace,
+			oneReplica: true,
 		},
+		// dependencies for the node-plugin
 		&yamlResourceNamespaced{
 			filename:  cephFSDirPath + cephFSNodePluginRBAC,
 			namespace: cephCSINamespace,
@@ -102,6 +102,11 @@ func createORDeleteCephfsResources(action kubectlAction) {
 			filename:  cephFSDirPath + cephFSNodePluginPSP,
 			namespace: cephCSINamespace,
 		},
+		// the node-plugin itself
+		&yamlResourceNamespaced{
+			filename:  cephFSDirPath + cephFSNodePlugin,
+			namespace: cephCSINamespace,
+		},
 	}
 
 	for _, r := range resources {
@@ -149,8 +154,8 @@ func validateSubvolumePath(f *framework.Framework, pvcName, pvcNamespace, fileSy
 	return nil
 }
 
-var _ = Describe("cephfs", func() {
-	f := framework.NewDefaultFramework("cephfs")
+var _ = Describe(cephfsType, func() {
+	f := framework.NewDefaultFramework(cephfsType)
 	var c clientset.Interface
 	// deploy CephFS CSI
 	BeforeEach(func() {
@@ -254,6 +259,12 @@ var _ = Describe("cephfs", func() {
 			snapshotPath := cephFSExamplePath + "snapshot.yaml"
 			appEphemeralPath := cephFSExamplePath + "pod-ephemeral.yaml"
 			pvcRWOPPath := cephFSExamplePath + "pvc-rwop.yaml"
+
+			metadataPool, getErr := getCephFSMetadataPoolName(f, fileSystemName)
+			if getErr != nil {
+				e2elog.Failf("failed getting cephFS metadata pool name: %v", getErr)
+			}
+
 			By("checking provisioner deployment is running", func() {
 				err := waitForDeploymentComplete(f.ClientSet, cephFSDeploymentName, cephCSINamespace, deployTimeout)
 				if err != nil {
@@ -304,12 +315,14 @@ var _ = Describe("cephfs", func() {
 						e2elog.Failf("failed to create application: %v", err)
 					}
 					validateSubvolumeCount(f, 1, fileSystemName, subvolumegroup)
+					validateOmapCount(f, 1, cephfsType, metadataPool, volumesType)
 					// delete pod
 					err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 					if err != nil {
 						e2elog.Failf("failed to delete application: %v", err)
 					}
 					validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+					validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 					err = deleteResource(cephFSExamplePath + "storageclass.yaml")
 					if err != nil {
 						e2elog.Failf("failed to delete CephFS storageclass: %v", err)
@@ -351,12 +364,14 @@ var _ = Describe("cephfs", func() {
 						e2elog.Failf("failed to create application: %v", err)
 					}
 					validateSubvolumeCount(f, 1, fileSystemName, subvolumegroup)
+					validateOmapCount(f, 1, cephfsType, metadataPool, volumesType)
 
 					err = validateRWOPPodCreation(f, pvc, app, baseAppName)
 					if err != nil {
 						e2elog.Failf("failed to validate RWOP pod creation: %v", err)
 					}
 					validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+					validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 					err = deleteResource(cephFSExamplePath + "storageclass.yaml")
 					if err != nil {
 						e2elog.Failf("failed to delete CephFS storageclass: %v", err)
@@ -409,6 +424,7 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, 1, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 1, cephfsType, metadataPool, volumesType)
 				// list subvolumes and check if one of them has the same prefix
 				foundIt := false
 				subvolumes, err := listCephFSSubVolumes(f, fileSystemName, subvolumegroup)
@@ -429,6 +445,7 @@ var _ = Describe("cephfs", func() {
 					e2elog.Failf("failed to delete PVC: %v", err)
 				}
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 				err = deleteResource(cephFSExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -648,6 +665,7 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, totalCount, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, volumesType)
 				// delete PVC and app
 				for i := 0; i < totalCount; i++ {
 					name := fmt.Sprintf("%s%d", f.UniqueName, i)
@@ -658,6 +676,7 @@ var _ = Describe("cephfs", func() {
 
 				}
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 			})
 
 			By("check data persist after recreating pod", func() {
@@ -1072,6 +1091,8 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
 
 				wg.Add(totalCount)
 				// delete clone and app
@@ -1098,6 +1119,8 @@ var _ = Describe("cephfs", func() {
 
 				parentPVCCount := totalSubvolumes - totalCount
 				validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
+				validateOmapCount(f, parentPVCCount, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
 				// create clones from different snapshots and bind it to an
 				// app
 				wg.Add(totalCount)
@@ -1129,6 +1152,8 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
 
 				wg.Add(totalCount)
 				// delete snapshot
@@ -1176,6 +1201,8 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
+				validateOmapCount(f, parentPVCCount, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, 0, cephfsType, metadataPool, snapsType)
 				// delete parent pvc
 				err = deletePVCAndApp("", f, pvc, app)
 				if err != nil {
@@ -1183,6 +1210,8 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, 0, cephfsType, metadataPool, snapsType)
 			})
 
 			By("create a PVC-PVC clone and bind it to an app", func() {
@@ -1254,6 +1283,7 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
 
 				// delete parent pvc
 				err = deletePVCAndApp("", f, pvc, app)
@@ -1285,6 +1315,7 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 			})
 
 			By("Create ROX PVC and bind it to an app", func() {
@@ -1363,6 +1394,7 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 			})
 
 			By("clone PVC to a bigger size PVC", func() {
@@ -1376,8 +1408,19 @@ var _ = Describe("cephfs", func() {
 				}
 
 				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
 			})
 
+			// FIXME: in case NFS testing is done, prevent deletion
+			// of the CephFS filesystem and related pool. This can
+			// probably be addressed in a nicer way, making sure
+			// everything is tested, always.
+			if testNFS {
+				e2elog.Logf("skipping CephFS destructive tests, allow NFS to run")
+
+				return
+			}
+
 			// Make sure this should be last testcase in this file, because
 			// it deletes pool
 			By("Create a PVC and delete PVC when backend pool deleted", func() {

e2e/cephfs_helper.go

@@ -99,7 +99,6 @@ func createCephfsStorageClass(
 		}
 		sc.Parameters["clusterID"] = fsID
 	}
-	sc.Namespace = cephCSINamespace
 
 	timeout := time.Duration(deployTimeout) * time.Minute
 
@@ -145,9 +144,7 @@ func unmountCephFSVolume(f *framework.Framework, appName, pvcName string) error
 
 		return fmt.Errorf("failed to get pod: %w", err)
 	}
-	pvc, err := f.ClientSet.CoreV1().
-		PersistentVolumeClaims(f.UniqueName).
-		Get(context.TODO(), pvcName, metav1.GetOptions{})
+	pvc, err := getPersistentVolumeClaim(f.ClientSet, f.UniqueName, pvcName)
 	if err != nil {
 		e2elog.Logf("Error occurred getting PVC %s in namespace %s", pvcName, f.UniqueName)
 
@@ -157,7 +154,7 @@ func unmountCephFSVolume(f *framework.Framework, appName, pvcName string) error
 		"umount /var/lib/kubelet/pods/%s/volumes/kubernetes.io~csi/%s/mount",
 		pod.UID,
 		pvc.Spec.VolumeName)
-	_, stdErr, err := execCommandInDaemonsetPod(
+	stdErr, err := execCommandInDaemonsetPod(
 		f,
 		cmd,
 		cephFSDeamonSetName,

e2e/deployment.go

@@ -192,6 +192,9 @@ type ResourceDeployer interface {
 type yamlResource struct {
 	filename string
 
+	// namespace defaults to cephCSINamespace if not set
+	namespace string
+
 	// allowMissing prevents a failure in case the file is missing.
 	allowMissing bool
 }
@@ -206,7 +209,12 @@ func (yr *yamlResource) Do(action kubectlAction) error {
 		return fmt.Errorf("failed to read content from %q: %w", yr.filename, err)
 	}
 
-	err = retryKubectlInput(cephCSINamespace, action, string(data), deployTimeout)
+	ns := cephCSINamespace
+	if yr.namespace != "" {
+		ns = yr.namespace
+	}
+
+	err = retryKubectlInput(ns, action, string(data), deployTimeout)
 	if err != nil {
 		return fmt.Errorf("failed to %s resource %q: %w", action, yr.filename, err)
 	}
@@ -254,3 +262,37 @@ func (yrn *yamlResourceNamespaced) Do(action kubectlAction) error {
 
 	return nil
 }
+
+type rookNFSResource struct {
+	f           *framework.Framework
+	modules     []string
+	orchBackend string
+}
+
+func (rnr *rookNFSResource) Do(action kubectlAction) error {
+	if action != kubectlCreate {
+		// we won't disabled modules
+		return nil
+	}
+
+	for _, module := range rnr.modules {
+		cmd := fmt.Sprintf("ceph mgr module enable %s", module)
+		_, _, err := execCommandInToolBoxPod(rnr.f, cmd, rookNamespace)
+		if err != nil {
+			// depending on the Ceph/Rook version, modules are
+			// enabled by default
+			e2elog.Logf("enabling module %q failed: %v", module, err)
+		}
+	}
+
+	if rnr.orchBackend != "" {
+		// this is not required for all Rook versions, allow failing
+		cmd := fmt.Sprintf("ceph orch set backend %s", rnr.orchBackend)
+		_, _, err := execCommandInToolBoxPod(rnr.f, cmd, rookNamespace)
+		if err != nil {
+			e2elog.Logf("setting orch backend %q failed: %v", rnr.orchBackend, err)
+		}
+	}
+
+	return nil
+}

e2e/e2e_test.go

@@ -36,8 +36,10 @@ func init() {
 	flag.IntVar(&deployTimeout, "deploy-timeout", 10, "timeout to wait for created kubernetes resources")
 	flag.BoolVar(&deployCephFS, "deploy-cephfs", true, "deploy cephFS csi driver")
 	flag.BoolVar(&deployRBD, "deploy-rbd", true, "deploy rbd csi driver")
+	flag.BoolVar(&deployNFS, "deploy-nfs", false, "deploy nfs csi driver")
 	flag.BoolVar(&testCephFS, "test-cephfs", true, "test cephFS csi driver")
 	flag.BoolVar(&testRBD, "test-rbd", true, "test rbd csi driver")
+	flag.BoolVar(&testNFS, "test-nfs", false, "test nfs csi driver")
 	flag.BoolVar(&helmTest, "helm-test", false, "tests running on deployment via helm")
 	flag.BoolVar(&upgradeTesting, "upgrade-testing", false, "perform upgrade testing")
 	flag.StringVar(&upgradeVersion, "upgrade-version", "v3.5.1", "target version for upgrade testing")
@@ -46,6 +48,7 @@ func init() {
 	flag.BoolVar(&isOpenShift, "is-openshift", false, "disables certain checks on OpenShift")
 	flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use")
 	flag.StringVar(&clusterID, "clusterid", "", "Ceph cluster ID to use (defaults to `ceph fsid` detection)")
+	flag.StringVar(&nfsDriverName, "nfs-driver", "nfs.csi.ceph.com", "name of the driver for NFS-volumes")
 	setDefaultKubeconfig()
 
 	// Register framework flags, then handle flags
@@ -75,4 +78,14 @@ func handleFlags() {
 	framework.RegisterClusterFlags(flag.CommandLine)
 	testing.Init()
 	flag.Parse()
+
+	// testNFS will automatically be enabled when testCephFS is enabled,
+	// this makes sure the NFS tests run in the CI where there are
+	// different jobs for CephFS and RBD. With a dedicated testNFS
+	// variable, it is still possible to only run the NFS tests, when both
+	// CephFS and RBD are disabled.
+	if testCephFS {
+		testNFS = testCephFS
+		deployNFS = deployCephFS
+	}
 }

e2e/errors.go

@@ -127,3 +127,33 @@ func isAlreadyExistsCLIError(err error) bool {
 
 	return true
 }
+
+// isNotFoundCLIError checks for "is not found" error from kubectl CLI.
+func isNotFoundCLIError(err error) bool {
+	if err == nil {
+		return false
+	}
+	// if multiple resources already exists. each error is separated by newline
+	stdErr := getStdErr(err.Error())
+	if stdErr == "" {
+		return false
+	}
+
+	stdErrs := strings.Split(stdErr, "\n")
+	for _, s := range stdErrs {
+		// If the string is just a new line continue
+		if strings.TrimSuffix(s, "\n") == "" {
+			continue
+		}
+		// Ignore warnings
+		if strings.Contains(s, "Warning") {
+			continue
+		}
+		// Resource not found error message
+		if !strings.Contains(s, "Error from server (NotFound)") {
+			return false
+		}
+	}
+
+	return true
+}

e2e/nfs.go

@@ -0,0 +1,992 @@
+/*
+Copyright 2022 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
+	. "github.com/onsi/ginkgo" // nolint
+	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
+)
+
+var (
+	nfsProvisioner     = "csi-nfsplugin-provisioner.yaml"
+	nfsProvisionerRBAC = "csi-provisioner-rbac.yaml"
+	nfsProvisionerPSP  = "csi-provisioner-psp.yaml"
+	nfsNodePlugin      = "csi-nfsplugin.yaml"
+	nfsNodePluginRBAC  = "csi-nodeplugin-rbac.yaml"
+	nfsNodePluginPSP   = "csi-nodeplugin-psp.yaml"
+	nfsRookCephNFS     = "rook-nfs.yaml"
+	nfsDeploymentName  = "csi-nfsplugin-provisioner"
+	nfsDeamonSetName   = "csi-nfs-node"
+	nfsDirPath         = "../deploy/nfs/kubernetes/"
+	nfsExamplePath     = examplePath + "nfs/"
+	nfsPoolName        = ".nfs"
+
+	// FIXME: some tests change the subvolumegroup to "e2e".
+	defaultSubvolumegroup = "csi"
+)
+
+func deployNFSPlugin(f *framework.Framework) {
+	// delete objects deployed by rook
+
+	err := deleteResource(nfsDirPath + nfsProvisionerRBAC)
+	if err != nil {
+		e2elog.Failf("failed to delete provisioner rbac %s: %v", nfsDirPath+nfsProvisionerRBAC, err)
+	}
+
+	err = deleteResource(nfsDirPath + nfsNodePluginRBAC)
+	if err != nil {
+		e2elog.Failf("failed to delete nodeplugin rbac %s: %v", nfsDirPath+nfsNodePluginRBAC, err)
+	}
+
+	// the pool should not be deleted, as it may contain configurations
+	// from non-e2e related CephNFS objects
+	err = createPool(f, nfsPoolName)
+	if err != nil {
+		e2elog.Failf("failed to create pool for NFS config %q: %v", nfsPoolName, err)
+	}
+
+	createORDeleteNFSResources(f, kubectlCreate)
+}
+
+func deleteNFSPlugin() {
+	createORDeleteNFSResources(nil, kubectlDelete)
+}
+
+func createORDeleteNFSResources(f *framework.Framework, action kubectlAction) {
+	resources := []ResourceDeployer{
+		// shared resources
+		&yamlResource{
+			filename:     nfsDirPath + csiDriverObject,
+			allowMissing: true,
+		},
+		&yamlResource{
+			filename:     examplePath + cephConfconfigMap,
+			allowMissing: true,
+		},
+		// dependencies for provisioner
+		&yamlResourceNamespaced{
+			filename:  nfsDirPath + nfsProvisionerRBAC,
+			namespace: cephCSINamespace,
+		},
+		&yamlResourceNamespaced{
+			filename:  nfsDirPath + nfsProvisionerPSP,
+			namespace: cephCSINamespace,
+		},
+		// the provisioner itself
+		&yamlResourceNamespaced{
+			filename:   nfsDirPath + nfsProvisioner,
+			namespace:  cephCSINamespace,
+			oneReplica: true,
+		},
+		// dependencies for the node-plugin
+		&yamlResourceNamespaced{
+			filename:  nfsDirPath + nfsNodePluginRBAC,
+			namespace: cephCSINamespace,
+		},
+		&yamlResourceNamespaced{
+			filename:  nfsDirPath + nfsNodePluginPSP,
+			namespace: cephCSINamespace,
+		},
+		// the node-plugin itself
+		&yamlResourceNamespaced{
+			filename:  nfsDirPath + nfsNodePlugin,
+			namespace: cephCSINamespace,
+		},
+		// NFS-export management by Rook
+		&rookNFSResource{
+			f:           f,
+			modules:     []string{"rook", "nfs"},
+			orchBackend: "rook",
+		},
+		&yamlResourceNamespaced{
+			filename:  nfsExamplePath + nfsRookCephNFS,
+			namespace: rookNamespace,
+		},
+	}
+
+	for _, r := range resources {
+		err := r.Do(action)
+		if err != nil {
+			e2elog.Failf("failed to %s resource: %v", action, err)
+		}
+	}
+}
+
+func createNFSStorageClass(
+	c clientset.Interface,
+	f *framework.Framework,
+	enablePool bool,
+	params map[string]string) error {
+	scPath := fmt.Sprintf("%s/%s", nfsExamplePath, "storageclass.yaml")
+	sc, err := getStorageClass(scPath)
+	if err != nil {
+		return err
+	}
+	sc.Parameters["nfsCluster"] = "my-nfs"
+	sc.Parameters["server"] = "rook-ceph-nfs-my-nfs-a." + rookNamespace + ".svc.cluster.local"
+
+	// standard CephFS parameters
+	sc.Parameters["fsName"] = fileSystemName
+	sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephFSProvisionerSecretName
+
+	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/controller-expand-secret-name"] = cephFSProvisionerSecretName
+
+	sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/node-stage-secret-name"] = cephFSNodePluginSecretName
+
+	if enablePool {
+		sc.Parameters["pool"] = "myfs-replicated"
+	}
+
+	// overload any parameters that were passed
+	if params == nil {
+		// create an empty params, so that params["clusterID"] below
+		// does not panic
+		params = map[string]string{}
+	}
+	for param, value := range params {
+		sc.Parameters[param] = value
+	}
+
+	// fetch and set fsID from the cluster if not set in params
+	if _, found := params["clusterID"]; !found {
+		var fsID string
+		fsID, err = getClusterID(f)
+		if err != nil {
+			return fmt.Errorf("failed to get clusterID: %w", err)
+		}
+		sc.Parameters["clusterID"] = fsID
+	}
+
+	sc.Provisioner = nfsDriverName
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
+		if err != nil {
+			e2elog.Logf("error creating StorageClass %q: %v", sc.Name, err)
+			if apierrs.IsAlreadyExists(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to create StorageClass %q: %w", sc.Name, err)
+		}
+
+		return true, nil
+	})
+}
+
+// unmountNFSVolume unmounts a NFS volume mounted on a pod.
+func unmountNFSVolume(f *framework.Framework, appName, pvcName string) error {
+	pod, err := f.ClientSet.CoreV1().Pods(f.UniqueName).Get(context.TODO(), appName, metav1.GetOptions{})
+	if err != nil {
+		e2elog.Logf("Error occurred getting pod %s in namespace %s", appName, f.UniqueName)
+
+		return fmt.Errorf("failed to get pod: %w", err)
+	}
+	pvc, err := f.ClientSet.CoreV1().
+		PersistentVolumeClaims(f.UniqueName).
+		Get(context.TODO(), pvcName, metav1.GetOptions{})
+	if err != nil {
+		e2elog.Logf("Error occurred getting PVC %s in namespace %s", pvcName, f.UniqueName)
+
+		return fmt.Errorf("failed to get pvc: %w", err)
+	}
+	cmd := fmt.Sprintf(
+		"umount /var/lib/kubelet/pods/%s/volumes/kubernetes.io~csi/%s/mount",
+		pod.UID,
+		pvc.Spec.VolumeName)
+	stdErr, err := execCommandInDaemonsetPod(
+		f,
+		cmd,
+		nfsDeamonSetName,
+		pod.Spec.NodeName,
+		"nfs", // name of the container
+		cephCSINamespace)
+	if stdErr != "" {
+		e2elog.Logf("StdErr occurred: %s", stdErr)
+	}
+
+	return err
+}
+
+var _ = Describe("nfs", func() {
+	f := framework.NewDefaultFramework("nfs")
+	var c clientset.Interface
+	// deploy CephFS CSI
+	BeforeEach(func() {
+		if !testNFS || upgradeTesting || helmTest {
+			Skip("Skipping NFS E2E")
+		}
+		c = f.ClientSet
+		if deployNFS {
+			if cephCSINamespace != defaultNs {
+				err := createNamespace(c, cephCSINamespace)
+				if err != nil {
+					e2elog.Failf("failed to create namespace %s: %v", cephCSINamespace, err)
+				}
+			}
+			deployNFSPlugin(f)
+		}
+
+		// cephfs testing might have changed the default subvolumegroup
+		subvolumegroup = defaultSubvolumegroup
+		err := createConfigMap(nfsDirPath, f.ClientSet, f)
+		if err != nil {
+			e2elog.Failf("failed to create configmap: %v", err)
+		}
+		// create nfs provisioner secret
+		key, err := createCephUser(f, keyringCephFSProvisionerUsername, cephFSProvisionerCaps())
+		if err != nil {
+			e2elog.Failf("failed to create user %s: %v", keyringCephFSProvisionerUsername, err)
+		}
+		err = createCephfsSecret(f, cephFSProvisionerSecretName, keyringCephFSProvisionerUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create provisioner secret: %v", err)
+		}
+		// create nfs plugin secret
+		key, err = createCephUser(f, keyringCephFSNodePluginUsername, cephFSNodePluginCaps())
+		if err != nil {
+			e2elog.Failf("failed to create user %s: %v", keyringCephFSNodePluginUsername, err)
+		}
+		err = createCephfsSecret(f, cephFSNodePluginSecretName, keyringCephFSNodePluginUsername, key)
+		if err != nil {
+			e2elog.Failf("failed to create node secret: %v", err)
+		}
+	})
+
+	AfterEach(func() {
+		if !testNFS || upgradeTesting {
+			Skip("Skipping NFS E2E")
+		}
+		if CurrentGinkgoTestDescription().Failed {
+			// log pods created by helm chart
+			logsCSIPods("app=ceph-csi-nfs", c)
+			// log provisioner
+			logsCSIPods("app=csi-nfsplugin-provisioner", c)
+			// log node plugin
+			logsCSIPods("app=csi-nfs-node", c)
+
+			// log all details from the namespace where Ceph-CSI is deployed
+			framework.DumpAllNamespaceInfo(c, cephCSINamespace)
+		}
+		err := deleteConfigMap(nfsDirPath)
+		if err != nil {
+			e2elog.Failf("failed to delete configmap: %v", err)
+		}
+		err = c.CoreV1().
+			Secrets(cephCSINamespace).
+			Delete(context.TODO(), cephFSProvisionerSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete provisioner secret: %v", err)
+		}
+		err = c.CoreV1().
+			Secrets(cephCSINamespace).
+			Delete(context.TODO(), cephFSNodePluginSecretName, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Failf("failed to delete node secret: %v", err)
+		}
+		err = deleteResource(nfsExamplePath + "storageclass.yaml")
+		if err != nil {
+			e2elog.Failf("failed to delete storageclass: %v", err)
+		}
+		if deployNFS {
+			deleteNFSPlugin()
+			if cephCSINamespace != defaultNs {
+				err := deleteNamespace(c, cephCSINamespace)
+				if err != nil {
+					e2elog.Failf("failed to delete namespace %s: %v", cephCSINamespace, err)
+				}
+			}
+		}
+	})
+
+	Context("Test NFS CSI", func() {
+		if !testNFS {
+			return
+		}
+
+		It("Test NFS CSI", func() {
+			pvcPath := nfsExamplePath + "pvc.yaml"
+			appPath := nfsExamplePath + "pod.yaml"
+			appRWOPPath := nfsExamplePath + "pod-rwop.yaml"
+			pvcRWOPPath := nfsExamplePath + "pvc-rwop.yaml"
+			pvcSmartClonePath := nfsExamplePath + "pvc-clone.yaml"
+			appSmartClonePath := nfsExamplePath + "pod-clone.yaml"
+			pvcClonePath := nfsExamplePath + "pvc-restore.yaml"
+			appClonePath := nfsExamplePath + "pod-restore.yaml"
+			snapshotPath := nfsExamplePath + "snapshot.yaml"
+
+			metadataPool, getErr := getCephFSMetadataPoolName(f, fileSystemName)
+			if getErr != nil {
+				e2elog.Failf("failed getting cephFS metadata pool name: %v", getErr)
+			}
+
+			By("checking provisioner deployment is running", func() {
+				err := waitForDeploymentComplete(f.ClientSet, nfsDeploymentName, cephCSINamespace, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for deployment %s: %v", nfsDeploymentName, err)
+				}
+			})
+
+			By("checking nodeplugin deamonset pods are running", func() {
+				err := waitForDaemonSets(nfsDeamonSetName, cephCSINamespace, f.ClientSet, deployTimeout)
+				if err != nil {
+					e2elog.Failf("timeout waiting for daemonset %s: %v", nfsDeamonSetName, err)
+				}
+			})
+
+			By("verify RWOP volume support", func() {
+				if k8sVersionGreaterEquals(f.ClientSet, 1, 22) {
+					err := createNFSStorageClass(f.ClientSet, f, false, nil)
+					if err != nil {
+						e2elog.Failf("failed to create CephFS storageclass: %v", err)
+					}
+					pvc, err := loadPVC(pvcRWOPPath)
+					if err != nil {
+						e2elog.Failf("failed to load PVC: %v", err)
+					}
+					pvc.Namespace = f.UniqueName
+
+					// create application
+					app, err := loadApp(appRWOPPath)
+					if err != nil {
+						e2elog.Failf("failed to load application: %v", err)
+					}
+					app.Namespace = f.UniqueName
+					baseAppName := app.Name
+
+					err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+					if err != nil {
+						if rwopMayFail(err) {
+							e2elog.Logf("RWOP is not supported: %v", err)
+
+							return
+						}
+						e2elog.Failf("failed to create PVC: %v", err)
+					}
+					err = createApp(f.ClientSet, app, deployTimeout)
+					if err != nil {
+						e2elog.Failf("failed to create application: %v", err)
+					}
+					validateSubvolumeCount(f, 1, fileSystemName, defaultSubvolumegroup)
+
+					err = validateRWOPPodCreation(f, pvc, app, baseAppName)
+					if err != nil {
+						e2elog.Failf("failed to validate RWOP pod creation: %v", err)
+					}
+					validateSubvolumeCount(f, 0, fileSystemName, defaultSubvolumegroup)
+					err = deleteResource(nfsExamplePath + "storageclass.yaml")
+					if err != nil {
+						e2elog.Failf("failed to delete CephFS storageclass: %v", err)
+					}
+				}
+			})
+
+			By("create a storageclass with pool and a PVC then bind it to an app", func() {
+				err := createNFSStorageClass(f.ClientSet, f, false, nil)
+				if err != nil {
+					e2elog.Failf("failed to create CephFS storageclass: %v", err)
+				}
+				err = validatePVCAndAppBinding(pvcPath, appPath, f)
+				if err != nil {
+					e2elog.Failf("failed to validate CephFS pvc and application binding: %v", err)
+				}
+				err = deleteResource(nfsExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete CephFS storageclass: %v", err)
+				}
+			})
+
+			By("create a PVC and bind it to an app", func() {
+				err := createNFSStorageClass(f.ClientSet, f, false, nil)
+				if err != nil {
+					e2elog.Failf("failed to create CephFS storageclass: %v", err)
+				}
+				err = validatePVCAndAppBinding(pvcPath, appPath, f)
+				if err != nil {
+					e2elog.Failf("failed to validate CephFS pvc and application  binding: %v", err)
+				}
+			})
+
+			By("create a PVC and bind it to an app with normal user", func() {
+				err := validateNormalUserPVCAccess(pvcPath, f)
+				if err != nil {
+					e2elog.Failf("failed to validate normal user CephFS pvc and application binding: %v", err)
+				}
+			})
+
+			By("create/delete multiple PVCs and Apps", func() {
+				totalCount := 2
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				app.Namespace = f.UniqueName
+				// create PVC and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err = createPVCAndApp(name, f, pvc, app, deployTimeout)
+					if err != nil {
+						e2elog.Failf("failed to create PVC or application: %v", err)
+					}
+					err = validateSubvolumePath(f, pvc.Name, pvc.Namespace, fileSystemName, defaultSubvolumegroup)
+					if err != nil {
+						e2elog.Failf("failed to validate subvolumePath: %v", err)
+					}
+				}
+
+				validateSubvolumeCount(f, totalCount, fileSystemName, defaultSubvolumegroup)
+				// delete PVC and app
+				for i := 0; i < totalCount; i++ {
+					name := fmt.Sprintf("%s%d", f.UniqueName, i)
+					err = deletePVCAndApp(name, f, pvc, app)
+					if err != nil {
+						e2elog.Failf("failed to delete PVC or application: %v", err)
+					}
+
+				}
+				validateSubvolumeCount(f, 0, fileSystemName, defaultSubvolumegroup)
+			})
+
+			By("check data persist after recreating pod", func() {
+				err := checkDataPersist(pvcPath, appPath, f)
+				if err != nil {
+					e2elog.Failf("failed to check data persist in pvc: %v", err)
+				}
+			})
+
+			By("Create PVC, bind it to an app, unmount volume and check app deletion", func() {
+				pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC or application: %v", err)
+				}
+
+				err = unmountNFSVolume(f, app.Name, pvc.Name)
+				if err != nil {
+					e2elog.Failf("failed to unmount volume: %v", err)
+				}
+
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC or application: %v", err)
+				}
+			})
+
+			By("Mount pvc as readonly in pod", func() {
+				// create PVC and bind it to an app
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvc.Namespace = f.UniqueName
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+
+				app.Namespace = f.UniqueName
+				label := map[string]string{
+					"app": app.Name,
+				}
+				app.Labels = label
+				app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+				app.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true
+				err = createPVCAndApp("", f, pvc, app, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC or application: %v", err)
+				}
+
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("app=%s", app.Name),
+				}
+
+				filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+				_, stdErr := execCommandInPodAndAllowFail(
+					f,
+					fmt.Sprintf("echo 'Hello World' > %s", filePath),
+					app.Namespace,
+					&opt)
+				readOnlyErr := fmt.Sprintf("cannot create %s: Read-only file system", filePath)
+				if !strings.Contains(stdErr, readOnlyErr) {
+					e2elog.Failf(stdErr)
+				}
+
+				// delete PVC and app
+				err = deletePVCAndApp("", f, pvc, app)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC or application: %v", err)
+				}
+			})
+
+			// delete nfs provisioner secret
+			err := deleteCephUser(f, keyringCephFSProvisionerUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s: %v", keyringCephFSProvisionerUsername, err)
+			}
+			// delete nfs plugin secret
+			err = deleteCephUser(f, keyringCephFSNodePluginUsername)
+			if err != nil {
+				e2elog.Failf("failed to delete user %s: %v", keyringCephFSNodePluginUsername, err)
+			}
+
+			By("Resize PVC and check application directory size", func() {
+				err := resizePVCAndValidateSize(pvcPath, appPath, f)
+				if err != nil {
+					e2elog.Failf("failed to resize PVC: %v", err)
+				}
+			})
+
+			By("create a PVC clone and bind it to an app", func() {
+				var wg sync.WaitGroup
+				totalCount := 3
+				wgErrs := make([]error, totalCount)
+				chErrs := make([]error, totalCount)
+				// totalSubvolumes represents the subvolumes in backend
+				// always totalCount+parentPVC
+				totalSubvolumes := totalCount + 1
+				wg.Add(totalCount)
+				err := createNFSSnapshotClass(f)
+				if err != nil {
+					e2elog.Failf("failed to delete NFS snapshotclass: %v", err)
+				}
+				defer func() {
+					err = deleteNFSSnapshotClass()
+					if err != nil {
+						e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err)
+					}
+				}()
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				pvc.Namespace = f.UniqueName
+				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC: %v", err)
+				}
+
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+
+				app.Namespace = f.UniqueName
+				app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+				label := make(map[string]string)
+				label[appKey] = appLabel
+				app.Labels = label
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+				}
+				checkSum, err := writeDataAndCalChecksum(app, &opt, f)
+				if err != nil {
+					e2elog.Failf("failed to calculate checksum: %v", err)
+				}
+
+				snap := getSnapshot(snapshotPath)
+				snap.Namespace = f.UniqueName
+				snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
+				// create snapshot
+				for i := 0; i < totalCount; i++ {
+					go func(n int, s snapapi.VolumeSnapshot) {
+						s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
+						wgErrs[n] = createSnapshot(&s, deployTimeout)
+						wg.Done()
+					}(i, snap)
+				}
+				wg.Wait()
+
+				failed := 0
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to create snapshot (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("creating snapshots failed, %d errors were logged", failed)
+				}
+
+				pvcClone, err := loadPVC(pvcClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				appClone, err := loadApp(appClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				pvcClone.Namespace = f.UniqueName
+				appClone.Namespace = f.UniqueName
+				pvcClone.Spec.DataSource.Name = fmt.Sprintf("%s%d", f.UniqueName, 0)
+				appClone.Labels = label
+
+				// create multiple PVC from same snapshot
+				wg.Add(totalCount)
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+						if wgErrs[n] == nil {
+							err = validateSubvolumePath(f, p.Name, p.Namespace, fileSystemName, subvolumegroup)
+							if err != nil {
+								wgErrs[n] = err
+							}
+							filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+							var checkSumClone string
+							e2elog.Logf("Calculating checksum clone for filepath %s", filePath)
+							checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
+							e2elog.Logf("checksum for clone is %s", checkSumClone)
+							if chErrs[n] != nil {
+								e2elog.Logf("Failed calculating checksum clone %s", chErrs[n])
+							}
+							if checkSumClone != checkSum {
+								e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
+							}
+						}
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				for i, err := range chErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
+				}
+
+				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
+
+				wg.Add(totalCount)
+				// delete clone and app
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						p.Spec.DataSource.Name = name
+						wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				parentPVCCount := totalSubvolumes - totalCount
+				validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
+				validateOmapCount(f, parentPVCCount, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
+				// create clones from different snapshots and bind it to an app
+				wg.Add(totalCount)
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						p.Spec.DataSource.Name = name
+						wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+						if wgErrs[n] == nil {
+							err = validateSubvolumePath(f, p.Name, p.Namespace, fileSystemName, subvolumegroup)
+							if err != nil {
+								wgErrs[n] = err
+							}
+							filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+							var checkSumClone string
+							e2elog.Logf("Calculating checksum clone for filepath %s", filePath)
+							checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
+							e2elog.Logf("checksum for clone is %s", checkSumClone)
+							if chErrs[n] != nil {
+								e2elog.Logf("Failed calculating checksum clone %s", chErrs[n])
+							}
+							if checkSumClone != checkSum {
+								e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
+							}
+						}
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to create PVC and app (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("creating PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				for i, err := range chErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
+				}
+
+				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, totalCount, cephfsType, metadataPool, snapsType)
+
+				wg.Add(totalCount)
+				// delete snapshot
+				for i := 0; i < totalCount; i++ {
+					go func(n int, s snapapi.VolumeSnapshot) {
+						s.Name = fmt.Sprintf("%s%d", f.UniqueName, n)
+						wgErrs[n] = deleteSnapshot(&s, deployTimeout)
+						wg.Done()
+					}(i, snap)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to delete snapshot (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("deleting snapshots failed, %d errors were logged", failed)
+				}
+
+				wg.Add(totalCount)
+				// delete clone and app
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						p.Spec.DataSource.Name = name
+						wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to delete PVC and app (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				validateSubvolumeCount(f, parentPVCCount, fileSystemName, subvolumegroup)
+				validateOmapCount(f, parentPVCCount, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, 0, cephfsType, metadataPool, snapsType)
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC or application: %v", err)
+				}
+
+				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
+				validateOmapCount(f, 0, cephfsType, metadataPool, snapsType)
+			})
+
+			By("create a PVC-PVC clone and bind it to an app", func() {
+				var wg sync.WaitGroup
+				totalCount := 3
+				wgErrs := make([]error, totalCount)
+				chErrs := make([]error, totalCount)
+				// totalSubvolumes represents the subvolumes in backend
+				// always totalCount+parentPVC
+				totalSubvolumes := totalCount + 1
+				pvc, err := loadPVC(pvcPath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+
+				pvc.Namespace = f.UniqueName
+				err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to create PVC: %v", err)
+				}
+				app, err := loadApp(appPath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				app.Namespace = f.UniqueName
+				app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
+				label := make(map[string]string)
+				label[appKey] = appLabel
+				app.Labels = label
+				opt := metav1.ListOptions{
+					LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
+				}
+				checkSum, err := writeDataAndCalChecksum(app, &opt, f)
+				if err != nil {
+					e2elog.Failf("failed to calculate checksum: %v", err)
+				}
+
+				pvcClone, err := loadPVC(pvcSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load PVC: %v", err)
+				}
+				pvcClone.Spec.DataSource.Name = pvc.Name
+				pvcClone.Namespace = f.UniqueName
+				appClone, err := loadApp(appSmartClonePath)
+				if err != nil {
+					e2elog.Failf("failed to load application: %v", err)
+				}
+				appClone.Namespace = f.UniqueName
+				appClone.Labels = label
+				wg.Add(totalCount)
+				// create clone and bind it to an app
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
+						if wgErrs[n] == nil {
+							filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
+							var checkSumClone string
+							e2elog.Logf("Calculating checksum clone for filepath %s", filePath)
+							checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
+							e2elog.Logf("checksum for clone is %s", checkSumClone)
+							if chErrs[n] != nil {
+								e2elog.Logf("Failed calculating checksum clone %s", chErrs[n])
+							}
+							if checkSumClone != checkSum {
+								e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
+							}
+						}
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				failed := 0
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to create PVC or application (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				for i, err := range chErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
+				}
+
+				validateSubvolumeCount(f, totalSubvolumes, fileSystemName, subvolumegroup)
+				validateOmapCount(f, totalSubvolumes, cephfsType, metadataPool, volumesType)
+
+				// delete parent pvc
+				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
+				if err != nil {
+					e2elog.Failf("failed to delete PVC or application: %v", err)
+				}
+
+				wg.Add(totalCount)
+				// delete clone and app
+				for i := 0; i < totalCount; i++ {
+					go func(n int, p v1.PersistentVolumeClaim, a v1.Pod) {
+						name := fmt.Sprintf("%s%d", f.UniqueName, n)
+						p.Spec.DataSource.Name = name
+						wgErrs[n] = deletePVCAndApp(name, f, &p, &a)
+						wg.Done()
+					}(i, *pvcClone, *appClone)
+				}
+				wg.Wait()
+
+				for i, err := range wgErrs {
+					if err != nil {
+						// not using Failf() as it aborts the test and does not log other errors
+						e2elog.Logf("failed to delete PVC or application (%s%d): %v", f.UniqueName, i, err)
+						failed++
+					}
+				}
+				if failed != 0 {
+					e2elog.Failf("deleting PVCs and apps failed, %d errors were logged", failed)
+				}
+
+				validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup)
+				validateOmapCount(f, 0, cephfsType, metadataPool, volumesType)
+			})
+		})
+	})
+})

e2e/pod.go

@@ -91,13 +91,31 @@ func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
 }
 
 func findPodAndContainerName(f *framework.Framework, ns, cn string, opt *metav1.ListOptions) (string, string, error) {
-	podList, err := f.PodClientNS(ns).List(context.TODO(), *opt)
-	if err != nil {
-		return "", "", err
-	}
+	timeout := time.Duration(deployTimeout) * time.Minute
 
-	if len(podList.Items) == 0 {
-		return "", "", errors.New("podlist is empty")
+	var (
+		podList *v1.PodList
+		listErr error
+	)
+	err := wait.PollImmediate(poll, timeout, func() (bool, error) {
+		podList, listErr = f.PodClientNS(ns).List(context.TODO(), *opt)
+		if listErr != nil {
+			if isRetryableAPIError(listErr) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to list Pods: %w", listErr)
+		}
+
+		if len(podList.Items) == 0 {
+			// retry in case the pods have not been (re)started yet
+			return false, nil
+		}
+
+		return true, nil
+	})
+	if err != nil {
+		return "", "", fmt.Errorf("failed to find pod for %v: %w", opt, err)
 	}
 
 	if cn != "" {
@@ -137,13 +155,16 @@ func getCommandInPodOpts(
 	}, nil
 }
 
-// execCommandInDaemonsetPod executes commands inside given container of a daemonset pod on a particular node.
+// execCommandInDaemonsetPod executes commands inside given container of a
+// daemonset pod on a particular node.
+//
+// stderr is returned as a string, and err will be set on a failure.
 func execCommandInDaemonsetPod(
 	f *framework.Framework,
-	c, daemonsetName, nodeName, containerName, ns string) (string, string, error) {
+	c, daemonsetName, nodeName, containerName, ns string) (string, error) {
 	selector, err := getDaemonSetLabelSelector(f, ns, daemonsetName)
 	if err != nil {
-		return "", "", err
+		return "", err
 	}
 
 	opt := &metav1.ListOptions{
@@ -151,7 +172,7 @@ func execCommandInDaemonsetPod(
 	}
 	pods, err := listPods(f, ns, opt)
 	if err != nil {
-		return "", "", err
+		return "", err
 	}
 
 	podName := ""
@@ -161,7 +182,7 @@ func execCommandInDaemonsetPod(
 		}
 	}
 	if podName == "" {
-		return "", "", fmt.Errorf("%s daemonset pod on node %s in namespace %s not found", daemonsetName, nodeName, ns)
+		return "", fmt.Errorf("%s daemonset pod on node %s in namespace %s not found", daemonsetName, nodeName, ns)
 	}
 
 	cmd := []string{"/bin/sh", "-c", c}
@@ -174,7 +195,9 @@ func execCommandInDaemonsetPod(
 		CaptureStderr: true,
 	}
 
-	return f.ExecWithOptions(podOpt)
+	_ /* stdout */, stderr, err := f.ExecWithOptions(podOpt)
+
+	return stderr, err
 }
 
 // listPods returns slice of pods matching given ListOptions and namespace.

e2e/pvc.go

@@ -191,14 +191,60 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
 	})
 }
 
+// getPersistentVolumeClaim returns the PersistentVolumeClaim with the given
+// name in the given namespace and retries if there is any API error.
+func getPersistentVolumeClaim(c kubernetes.Interface, namespace, name string) (*v1.PersistentVolumeClaim, error) {
+	var pvc *v1.PersistentVolumeClaim
+	var err error
+	timeout := time.Duration(deployTimeout) * time.Minute
+	err = wait.PollImmediate(1*time.Second, timeout, func() (bool, error) {
+		pvc, err = c.CoreV1().PersistentVolumeClaims(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			e2elog.Logf("Error getting pvc %q in namespace %q: %v", name, namespace, err)
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to get pvc: %w", err)
+		}
+
+		return true, err
+	})
+
+	return pvc, err
+}
+
+// getPersistentVolume returns the PersistentVolume with the given
+// name and retries if there is any API error.
+func getPersistentVolume(c kubernetes.Interface, name string) (*v1.PersistentVolume, error) {
+	var pv *v1.PersistentVolume
+	var err error
+	timeout := time.Duration(deployTimeout) * time.Minute
+	err = wait.PollImmediate(1*time.Second, timeout, func() (bool, error) {
+		pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			e2elog.Logf("Error getting pv %q: %v", name, err)
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to get pv: %w", err)
+		}
+
+		return true, err
+	})
+
+	return pv, err
+}
+
 func getPVCAndPV(
 	c kubernetes.Interface,
 	pvcName, pvcNamespace string) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error) {
-	pvc, err := c.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(context.TODO(), pvcName, metav1.GetOptions{})
+	pvc, err := getPersistentVolumeClaim(c, pvcNamespace, pvcName)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get PVC: %w", err)
 	}
-	pv, err := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+	pv, err := getPersistentVolume(c, pvc.Spec.VolumeName)
 	if err != nil {
 		return pvc, nil, fmt.Errorf("failed to get PV: %w", err)
 	}
@@ -213,11 +259,11 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 	var err error
 	e2elog.Logf("Deleting PersistentVolumeClaim %v on namespace %v", name, nameSpace)
 
-	pvc, err = c.CoreV1().PersistentVolumeClaims(nameSpace).Get(context.TODO(), name, metav1.GetOptions{})
+	pvc, err = getPersistentVolumeClaim(c, nameSpace, name)
 	if err != nil {
 		return fmt.Errorf("failed to get pvc: %w", err)
 	}
-	pv, err := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvc.Spec.VolumeName, metav1.GetOptions{})
+	pv, err := getPersistentVolume(c, pvc.Spec.VolumeName)
 	if err != nil {
 		return fmt.Errorf("failed to get pv: %w", err)
 	}
@@ -265,9 +311,7 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
 // getBoundPV returns a PV details.
 func getBoundPV(client kubernetes.Interface, pvc *v1.PersistentVolumeClaim) (*v1.PersistentVolume, error) {
 	// Get new copy of the claim
-	claim, err := client.CoreV1().
-		PersistentVolumeClaims(pvc.Namespace).
-		Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+	claim, err := getPersistentVolumeClaim(client, pvc.Namespace, pvc.Name)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get pvc: %w", err)
 	}

e2e/rbd.go

@@ -131,6 +131,7 @@ func deleteRBDPlugin() {
 
 func createORDeleteRbdResources(action kubectlAction) {
 	resources := []ResourceDeployer{
+		// shared resources
 		&yamlResource{
 			filename:     rbdDirPath + csiDriverObject,
 			allowMissing: true,
@@ -139,12 +140,7 @@ func createORDeleteRbdResources(action kubectlAction) {
 			filename:     examplePath + cephConfconfigMap,
 			allowMissing: true,
 		},
-		&yamlResourceNamespaced{
-			filename:       rbdDirPath + rbdProvisioner,
-			namespace:      cephCSINamespace,
-			oneReplica:     true,
-			enableTopology: true,
-		},
+		// dependencies for provisioner
 		&yamlResourceNamespaced{
 			filename:  rbdDirPath + rbdProvisionerRBAC,
 			namespace: cephCSINamespace,
@@ -153,11 +149,14 @@ func createORDeleteRbdResources(action kubectlAction) {
 			filename:  rbdDirPath + rbdProvisionerPSP,
 			namespace: cephCSINamespace,
 		},
+		// the provisioner itself
 		&yamlResourceNamespaced{
-			filename:    rbdDirPath + rbdNodePlugin,
-			namespace:   cephCSINamespace,
-			domainLabel: nodeRegionLabel + "," + nodeZoneLabel,
+			filename:       rbdDirPath + rbdProvisioner,
+			namespace:      cephCSINamespace,
+			oneReplica:     true,
+			enableTopology: true,
 		},
+		// dependencies for the node-plugin
 		&yamlResourceNamespaced{
 			filename:  rbdDirPath + rbdNodePluginRBAC,
 			namespace: cephCSINamespace,
@@ -166,6 +165,12 @@ func createORDeleteRbdResources(action kubectlAction) {
 			filename:  rbdDirPath + rbdNodePluginPSP,
 			namespace: cephCSINamespace,
 		},
+		// the node-plugin itself
+		&yamlResourceNamespaced{
+			filename:    rbdDirPath + rbdNodePlugin,
+			namespace:   cephCSINamespace,
+			domainLabel: nodeRegionLabel + "," + nodeZoneLabel,
+		},
 	}
 
 	for _, r := range resources {
@@ -191,8 +196,23 @@ func validateRBDImageCount(f *framework.Framework, count int, pool string) {
 	}
 }
 
+func formatImageMetaGetCmd(pool, image, key string) string {
+	return fmt.Sprintf("rbd image-meta get %s --image=%s %s", rbdOptions(pool), image, key)
+}
+
+// checkGetKeyError check for error conditions returned by get image-meta key,
+// returns true if key exists.
+func checkGetKeyError(err error, stdErr string) bool {
+	if err == nil || !strings.Contains(err.Error(), "command terminated with exit code 2") ||
+		!strings.Contains(stdErr, "failed to get metadata") {
+		return true
+	}
+
+	return false
+}
+
 var _ = Describe("RBD", func() {
-	f := framework.NewDefaultFramework("rbd")
+	f := framework.NewDefaultFramework(rbdType)
 	var c clientset.Interface
 	var kernelRelease string
 	// deploy RBD CSI
@@ -353,6 +373,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 					//  Deleting the storageclass and secret created by helm
 					err = deleteResource(rbdExamplePath + "storageclass.yaml")
 					if err != nil {
@@ -383,6 +404,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				imageList, err := listRBDImages(f, defaultRBDPool)
 				if err != nil {
@@ -390,8 +412,7 @@ var _ = Describe("RBD", func() {
 				}
 
 				pvcName, stdErr, err := execCommandInToolBoxPod(f,
-					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
-						rbdOptions(defaultRBDPool), imageList[0], pvcNameKey),
+					formatImageMetaGetCmd(defaultRBDPool, imageList[0], pvcNameKey),
 					rookNamespace)
 				if err != nil || stdErr != "" {
 					e2elog.Failf("failed to get PVC name %s/%s %s: err=%v stdErr=%q",
@@ -403,8 +424,7 @@ var _ = Describe("RBD", func() {
 				}
 
 				pvcNamespace, stdErr, err := execCommandInToolBoxPod(f,
-					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
-						rbdOptions(defaultRBDPool), imageList[0], pvcNamespaceKey),
+					formatImageMetaGetCmd(defaultRBDPool, imageList[0], pvcNamespaceKey),
 					rookNamespace)
 				if err != nil || stdErr != "" {
 					e2elog.Failf("failed to get PVC namespace %s/%s %s: err=%v stdErr=%q",
@@ -414,11 +434,7 @@ var _ = Describe("RBD", func() {
 				if pvcNamespace != pvc.Namespace {
 					e2elog.Failf("expected pvcNamespace %q got %q", pvc.Namespace, pvcNamespace)
 				}
-
-				pvcObj, err := c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(
-					context.TODO(),
-					pvc.Name,
-					metav1.GetOptions{})
+				pvcObj, err := getPersistentVolumeClaim(c, pvc.Namespace, pvc.Name)
 				if err != nil {
 					e2elog.Logf("error getting pvc %q in namespace %q: %v", pvc.Name, pvc.Namespace, err)
 				}
@@ -426,8 +442,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Logf("pv name is empty %q in namespace %q: %v", pvc.Name, pvc.Namespace, err)
 				}
 				pvName, stdErr, err := execCommandInToolBoxPod(f,
-					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
-						rbdOptions(defaultRBDPool), imageList[0], pvNameKey),
+					formatImageMetaGetCmd(defaultRBDPool, imageList[0], pvNameKey),
 					rookNamespace)
 				if err != nil || stdErr != "" {
 					e2elog.Failf("failed to get PV name %s/%s %s: err=%v stdErr=%q",
@@ -443,6 +458,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to delete pvc: %v", err)
 				}
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("reattach the old PV to a new PVC and check if PVC metadata is updated on RBD image", func() {
@@ -458,6 +474,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				imageList, err := listRBDImages(f, defaultRBDPool)
 				if err != nil {
@@ -524,6 +541,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				pvcObj.Name = "rbd-pvc-new"
 				// unset the resource version as should not be set on objects to be created
@@ -535,6 +553,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				pvcName, stdErr, err = execCommandInToolBoxPod(f,
 					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
@@ -564,6 +583,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to delete pvc: %v", err)
 				}
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create a snapshot and check metadata on RBD snapshot image", func() {
@@ -584,6 +604,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				// delete pod as we should not create snapshot for in-use pvc
 				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 				if err != nil {
@@ -602,6 +623,8 @@ var _ = Describe("RBD", func() {
 				// parent PVC + snapshot
 				totalImages := 2
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 				imageList, err := listRBDImages(f, defaultRBDPool)
 				if err != nil {
@@ -652,6 +675,33 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("expected volSnapContentName %q got %q", content.Name, volSnapContentName)
 				}
 
+				// make sure we had unset the PVC metadata on the rbd image created
+				// for the snapshot
+				pvcName, stdErr, err := execCommandInToolBoxPod(f,
+					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
+						rbdOptions(defaultRBDPool), imageList[0], pvcNameKey),
+					rookNamespace)
+				if checkGetKeyError(err, stdErr) {
+					e2elog.Failf("PVC name found on %s/%s %s=%s: err=%v stdErr=%q",
+						rbdOptions(defaultRBDPool), imageList[0], pvcNameKey, pvcName, err, stdErr)
+				}
+				pvcNamespace, stdErr, err := execCommandInToolBoxPod(f,
+					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
+						rbdOptions(defaultRBDPool), imageList[0], pvcNamespaceKey),
+					rookNamespace)
+				if checkGetKeyError(err, stdErr) {
+					e2elog.Failf("PVC namespace found on %s/%s %s=%s: err=%v stdErr=%q",
+						rbdOptions(defaultRBDPool), imageList[0], pvcNamespaceKey, pvcNamespace, err, stdErr)
+				}
+				pvName, stdErr, err := execCommandInToolBoxPod(f,
+					fmt.Sprintf("rbd image-meta get %s --image=%s %s",
+						rbdOptions(defaultRBDPool), imageList[0], pvNameKey),
+					rookNamespace)
+				if checkGetKeyError(err, stdErr) {
+					e2elog.Failf("PV name found on %s/%s %s=%s: err=%v stdErr=%q",
+						rbdOptions(defaultRBDPool), imageList[0], pvNameKey, pvName, err, stdErr)
+				}
+
 				err = deleteSnapshot(&snap, deployTimeout)
 				if err != nil {
 					e2elog.Failf("failed to delete snapshot: %v", err)
@@ -661,6 +711,7 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to delete pvc: %v", err)
 				}
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("verify generic ephemeral volume support", func() {
@@ -678,12 +729,14 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 					err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 					if err != nil {
 						e2elog.Failf("failed to delete application: %v", err)
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 					// validate images in trash
 					err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 					if err != nil {
@@ -703,6 +756,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				// Block PVC resize
 				err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
@@ -741,6 +795,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = tearDownMigrationSetup(f)
 				if err != nil {
 					e2elog.Failf("failed to tear down migration setup: %v", err)
@@ -758,6 +813,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create a PVC and bind it to an app", func() {
@@ -767,6 +823,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create a PVC and bind it to an app with normal user", func() {
@@ -776,6 +833,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create a Block mode RWOP PVC and bind it to more than one app", func() {
@@ -803,6 +861,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 					err = createApp(f.ClientSet, app, deployTimeout)
 					if err != nil {
@@ -814,6 +873,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				}
 			})
 
@@ -842,6 +902,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 					err = createApp(f.ClientSet, app, deployTimeout)
 					if err != nil {
@@ -853,6 +914,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				}
 			})
 
@@ -888,6 +950,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create an erasure coded PVC and validate snapshot restore", func() {
@@ -911,6 +974,7 @@ var _ = Describe("RBD", func() {
 					appPath,
 					pvcSmartClonePath,
 					appSmartClonePath,
+					defaultSCName,
 					erasureCodedPool,
 					noKMS,
 					noPVCValidation,
@@ -946,6 +1010,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -981,6 +1046,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1019,6 +1085,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 					// FileSystem PVC resize
 					err = resizePVCAndValidateSize(pvcPath, appPath, f)
@@ -1027,6 +1094,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 					err = deleteResource(rbdExamplePath + "storageclass.yaml")
 					if err != nil {
 						e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1063,6 +1131,7 @@ var _ = Describe("RBD", func() {
 						}
 						// validate created backend rbd images
 						validateRBDImageCount(f, 0, defaultRBDPool)
+						validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 						err = deleteResource(rbdExamplePath + "storageclass.yaml")
 						if err != nil {
 							e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1104,6 +1173,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 					if util.CheckKernelSupport(kernelRelease, deepFlattenSupport) {
 						app, aErr := loadApp(appPath)
@@ -1129,6 +1199,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				})
 
 			By("create PVC with layering,deep-flatten image-features and bind it to an app",
@@ -1161,6 +1232,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 					// checking the minimal kernel version for fast-diff as its
 					// higher kernel version than other default image features.
@@ -1188,6 +1260,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				})
 
 			By("create PVC with journaling,fast-diff image-features and bind it to an app using rbd-nbd mounter",
@@ -1217,6 +1290,7 @@ var _ = Describe("RBD", func() {
 						}
 						// validate created backend rbd images
 						validateRBDImageCount(f, 0, defaultRBDPool)
+						validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 						err = deleteResource(rbdExamplePath + "storageclass.yaml")
 						if err != nil {
 							e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1299,6 +1373,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				// validate images in trash
 				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 				if err != nil {
@@ -1352,6 +1427,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 				if err != nil {
 					e2elog.Failf("failed to delete application: %v", err)
@@ -1384,6 +1460,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 3, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
 
 				filePath := appClone.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
 				cmd := fmt.Sprintf("echo 'Hello World' > %s", filePath)
@@ -1421,6 +1498,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				// validate images in trash
 				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 				if err != nil {
@@ -1474,6 +1552,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 				if err != nil {
 					e2elog.Failf("failed to delete application: %v", err)
@@ -1508,6 +1587,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 3, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
 
 				devPath := appClone.Spec.Template.Spec.Containers[0].VolumeDevices[0].DevicePath
 				cmd := fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10", devPath)
@@ -1544,6 +1624,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				// validate images in trash
 				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 				if err != nil {
@@ -1618,6 +1699,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				selector, err := getDaemonSetLabelSelector(f, cephCSINamespace, rbdDaemonsetName)
 				if err != nil {
@@ -1711,6 +1793,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1748,6 +1831,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1779,6 +1863,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1812,6 +1897,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				// Block PVC resize
 				err = resizePVCAndValidateSize(rawPvcPath, rawAppPath, f)
@@ -1820,6 +1906,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -1850,6 +1937,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1893,6 +1981,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				// delete the Secret of the Tenant
 				err = c.CoreV1().Secrets(tenant).Delete(context.TODO(), token.Name, metav1.DeleteOptions{})
@@ -1936,6 +2025,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -1965,6 +2055,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -2004,6 +2095,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				// delete user secret
 				err = retryKubectlFile(namespace,
@@ -2056,6 +2148,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 					// delete user secret
 					err = retryKubectlFile(
@@ -2136,6 +2229,7 @@ var _ = Describe("RBD", func() {
 					appPath,
 					pvcSmartClonePath,
 					appSmartClonePath,
+					defaultSCName,
 					noDataPool,
 					noKMS,
 					noPVCValidation,
@@ -2217,6 +2311,67 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to delete storageclass: %v", err)
 				}
 
+				// validate created backend rbd images
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+			})
+
+			By("Validate PVC-PVC clone with different SC from vaultKMS to vaultTenantSAKMS", func() {
+				restoreSCName := "restore-sc"
+				err := deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+				scOpts := map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "vault-test",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				scOpts = map[string]string{
+					"encrypted":       "true",
+					"encryptionKMSID": "vault-tenant-sa-test",
+				}
+				err = createRBDStorageClass(f.ClientSet, f, restoreSCName, nil, scOpts, deletePolicy)
+				if err != nil {
+					e2elog.Failf("failed to create storageclass: %v", err)
+				}
+
+				err = createTenantServiceAccount(f.ClientSet, f.UniqueName)
+				if err != nil {
+					e2elog.Failf("failed to create ServiceAccount: %v", err)
+				}
+				defer deleteTenantServiceAccount(f.UniqueName)
+
+				validatePVCClone(1,
+					pvcPath,
+					appPath,
+					pvcSmartClonePath,
+					appSmartClonePath,
+					restoreSCName,
+					noDataPool,
+					secretsMetadataKMS,
+					isEncryptedPVC,
+					f)
+
+				err = retryKubectlArgs(cephCSINamespace, kubectlDelete, deployTimeout, "storageclass", restoreSCName)
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass %q: %v", restoreSCName, err)
+				}
+
+				err = deleteResource(rbdExamplePath + "storageclass.yaml")
+				if err != nil {
+					e2elog.Failf("failed to delete storageclass: %v", err)
+				}
+
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
 
@@ -2245,6 +2400,7 @@ var _ = Describe("RBD", func() {
 					appPath,
 					pvcSmartClonePath,
 					appSmartClonePath,
+					defaultSCName,
 					noDataPool,
 					secretsMetadataKMS,
 					isEncryptedPVC,
@@ -2279,6 +2435,7 @@ var _ = Describe("RBD", func() {
 					appPath,
 					pvcSmartClonePath,
 					appSmartClonePath,
+					defaultSCName,
 					noDataPool,
 					vaultKMS,
 					isEncryptedPVC,
@@ -2311,6 +2468,7 @@ var _ = Describe("RBD", func() {
 					rawAppPath,
 					pvcBlockSmartClonePath,
 					appBlockSmartClonePath,
+					defaultSCName,
 					noDataPool,
 					noKMS,
 					noPVCValidation,
@@ -2340,6 +2498,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, totalCount, defaultRBDPool)
+				validateOmapCount(f, totalCount, rbdType, defaultRBDPool, volumesType)
 				// delete PVC and app
 				for i := 0; i < totalCount; i++ {
 					name := fmt.Sprintf("%s%d", f.UniqueName, i)
@@ -2352,6 +2511,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("check data persist after recreating pod", func() {
@@ -2361,6 +2521,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("Resize Filesystem PVC and check application directory size", func() {
@@ -2389,6 +2550,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("Resize Block PVC and check Device size", func() {
@@ -2398,6 +2560,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("Test unmount after nodeplugin restart", func() {
@@ -2419,6 +2582,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				// delete rbd nodeplugin pods
 				err = deletePodWithLabel("app=csi-rbdplugin", cephCSINamespace, false)
 				if err != nil {
@@ -2436,6 +2600,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create PVC in storageClass with volumeNamePrefix", func() {
@@ -2467,6 +2632,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				// list RBD images and check if one of them has the same prefix
 				foundIt := false
 				images, err := listRBDImages(f, defaultRBDPool)
@@ -2489,6 +2655,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
@@ -2510,6 +2677,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("validate RBD static Block PVC", func() {
@@ -2519,6 +2687,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("validate failure of RBD static PVC without imageFeatures parameter", func() {
@@ -2528,6 +2697,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("validate mount options in app pod", func() {
@@ -2538,6 +2708,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("creating an app with a PVC, using a topology constrained StorageClass", func() {
@@ -2572,6 +2743,7 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create PVC and application: %v", err)
 				}
+
 				By("ensuring created PV has required node selector values populated")
 				err = checkPVSelectorValuesForPVC(f, pvc)
 				if err != nil {
@@ -2595,10 +2767,16 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to check csi journal in pool: %v", err)
 				}
 
+				err = deleteJournalInfoInPool(f, pvc, "replicapool")
+				if err != nil {
+					e2elog.Failf("failed to delete omap data: %v", err)
+				}
 				err = deletePVCAndApp("", f, pvc, app)
 				if err != nil {
 					e2elog.Failf("failed to delete PVC and application: %v", err)
 				}
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				By("checking if data pool parameter is honored", func() {
 					err = deleteResource(rbdExamplePath + "storageclass.yaml")
@@ -2633,11 +2811,17 @@ var _ = Describe("RBD", func() {
 						e2elog.Failf("failed to check data pool for image: %v", err)
 					}
 
+					err = deleteJournalInfoInPool(f, pvc, "replicapool")
+					if err != nil {
+						e2elog.Failf("failed to delete omap data: %v", err)
+					}
 					// cleanup and undo changes made by the test
 					err = deletePVCAndApp("", f, pvc, app)
 					if err != nil {
 						e2elog.Failf("failed to delete PVC and application: %v", err)
 					}
+					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				})
 
 				// cleanup and undo changes made by the test
@@ -2649,6 +2833,8 @@ var _ = Describe("RBD", func() {
 				if err != nil {
 					e2elog.Failf("failed to create storageclass: %v", err)
 				}
+				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			// Mount pvc to pod with invalid mount option,expected that
@@ -2685,6 +2871,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				// create an app and wait for 1 min for it to go to running state
 				err = createApp(f.ClientSet, app, 1)
@@ -2698,6 +2885,7 @@ var _ = Describe("RBD", func() {
 
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = deleteResource(rbdExamplePath + "storageclass.yaml")
 				if err != nil {
 					e2elog.Failf("failed to delete storageclass: %v", err)
@@ -2794,6 +2982,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 				// delete pod as we should not create snapshot for in-use pvc
 				err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 				if err != nil {
@@ -2812,6 +3001,8 @@ var _ = Describe("RBD", func() {
 				// parent PVC + snapshot
 				totalImages := 2
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 				pvcClone, err := loadPVC(pvcClonePath)
 				if err != nil {
 					e2elog.Failf("failed to load PVC: %v", err)
@@ -2828,6 +3019,8 @@ var _ = Describe("RBD", func() {
 				// parent pvc+ snapshot + clone
 				totalImages = 3
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 				appClone, err := loadApp(appClonePath)
 				if err != nil {
@@ -2896,6 +3089,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("validate PVC mounting if snapshot and parent PVC are deleted", func() {
@@ -2928,6 +3122,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				snap := getSnapshot(snapshotPath)
 				snap.Namespace = f.UniqueName
@@ -2941,6 +3136,8 @@ var _ = Describe("RBD", func() {
 				// parent PVC + snapshot
 				totalImages := 2
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 				pvcClone, err := loadPVC(pvcClonePath)
 				if err != nil {
 					e2elog.Failf("failed to load PVC: %v", err)
@@ -2953,6 +3150,8 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 				// create clone PVC
 				pvcClone.Namespace = f.UniqueName
@@ -2963,6 +3162,8 @@ var _ = Describe("RBD", func() {
 				// validate created backend rbd images = snapshot + clone
 				totalImages = 2
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 				// delete snapshot
 				err = deleteSnapshot(&snap, deployTimeout)
@@ -2973,6 +3174,8 @@ var _ = Describe("RBD", func() {
 				// validate created backend rbd images = clone
 				totalImages = 1
 				validateRBDImageCount(f, totalImages, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 
 				appClone, err := loadApp(appClonePath)
 				if err != nil {
@@ -2998,6 +3201,8 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 			})
 
 			By(
@@ -3062,6 +3267,7 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 1, defaultRBDPool)
+					validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 					for i := 0; i < snapChainDepth; i++ {
 						var pvcClone *v1.PersistentVolumeClaim
 						snap := getSnapshot(snapshotPath)
@@ -3077,6 +3283,8 @@ var _ = Describe("RBD", func() {
 						// parent PVC + snapshot
 						totalImages := 2
 						validateRBDImageCount(f, totalImages, defaultRBDPool)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 						pvcClone, err = loadPVC(pvcClonePath)
 						if err != nil {
 							e2elog.Failf("failed to load PVC: %v", err)
@@ -3089,6 +3297,8 @@ var _ = Describe("RBD", func() {
 						}
 						// validate created backend rbd images
 						validateRBDImageCount(f, 1, defaultRBDPool)
+						validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 						// create clone PVC
 						pvcClone.Name = fmt.Sprintf("%s-%d", pvcClone.Name, i)
@@ -3101,6 +3311,8 @@ var _ = Describe("RBD", func() {
 						// validate created backend rbd images = snapshot + clone
 						totalImages = 2
 						validateRBDImageCount(f, totalImages, defaultRBDPool)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 						// delete snapshot
 						err = deleteSnapshot(&snap, deployTimeout)
@@ -3111,6 +3323,8 @@ var _ = Describe("RBD", func() {
 						// validate created backend rbd images = clone
 						totalImages = 1
 						validateRBDImageCount(f, totalImages, defaultRBDPool)
+						validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+						validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 
 						app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcClone.Name
 						// create application
@@ -3133,6 +3347,8 @@ var _ = Describe("RBD", func() {
 					}
 					// validate created backend rbd images
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 				})
 
 			By("validate PVC Clone chained with depth 2", func() {
@@ -3185,6 +3401,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				for i := 0; i < cloneChainDepth; i++ {
 					var pvcClone *v1.PersistentVolumeClaim
@@ -3229,6 +3446,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("ensuring all operations will work within a rados namespace", func() {
@@ -3314,6 +3532,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				// Create a PVC and bind it to an app within the namesapce
 				err = validatePVCAndAppBinding(pvcPath, appPath, f)
@@ -3357,6 +3576,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				snap := getSnapshot(snapshotPath)
 				snap.Namespace = f.UniqueName
@@ -3366,6 +3586,8 @@ var _ = Describe("RBD", func() {
 					e2elog.Failf("failed to create snapshot: %v", err)
 				}
 				validateRBDImageCount(f, 2, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
 
 				err = validatePVCAndAppBinding(pvcClonePath, appClonePath, f)
 				if err != nil {
@@ -3377,12 +3599,16 @@ var _ = Describe("RBD", func() {
 				}
 				// as snapshot is deleted the image count should be one
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 
 				err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
 				if err != nil {
 					e2elog.Failf("failed to delete PVC: %v", err)
 				}
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, snapsType)
 
 				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 				if err != nil {
@@ -3449,6 +3675,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 1, defaultRBDPool)
+				validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
 
 				opt := metav1.ListOptions{
 					LabelSelector: fmt.Sprintf("app=%s", app.Name),
@@ -3472,6 +3699,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 			})
 
 			By("create a PVC and Bind it to an app for mapped rbd image with options", func() {
@@ -3517,6 +3745,7 @@ var _ = Describe("RBD", func() {
 				}
 				// validate created backend rbd images
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 				err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
 				if err != nil {
 					e2elog.Failf("failed to create storageclass : %v", err)
@@ -3526,6 +3755,7 @@ var _ = Describe("RBD", func() {
 			By("validate image deletion when it is moved to trash", func() {
 				// make sure pool is empty
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				err := createRBDSnapshotClass(f)
 				if err != nil {
@@ -3619,6 +3849,7 @@ var _ = Describe("RBD", func() {
 				}
 
 				validateRBDImageCount(f, 0, defaultRBDPool)
+				validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 
 				err = waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 				if err != nil {
@@ -3730,6 +3961,7 @@ var _ = Describe("RBD", func() {
 
 				By("validate image deletion", func() {
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 					err := waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 					if err != nil {
 						e2elog.Failf("failed to validate rbd images in pool %s trash: %v", defaultRBDPool, err)
@@ -3801,6 +4033,7 @@ var _ = Describe("RBD", func() {
 
 				By("validate image deletion", func() {
 					validateRBDImageCount(f, 0, defaultRBDPool)
+					validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
 					err := waitToRemoveImagesFromTrash(f, defaultRBDPool, deployTimeout)
 					if err != nil {
 						e2elog.Failf("failed to validate rbd images in pool %s trash: %v", defaultRBDPool, err)

e2e/rbd_helper.go

@@ -149,7 +149,6 @@ func createRBDStorageClass(
 			delete(sc.Parameters, k)
 		}
 	}
-	sc.Namespace = cephCSINamespace
 
 	if scOptions["volumeBindingMode"] == "WaitForFirstConsumer" {
 		value := scv1.VolumeBindingWaitForFirstConsumer

e2e/resize.go

@@ -36,10 +36,7 @@ import (
 func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size string, t int) error {
 	pvcName := pvc.Name
 	pvcNamespace := pvc.Namespace
-
-	updatedPVC, err := c.CoreV1().
-		PersistentVolumeClaims(pvcNamespace).
-		Get(context.TODO(), pvcName, metav1.GetOptions{})
+	updatedPVC, err := getPersistentVolumeClaim(c, pvcNamespace, pvcName)
 	if err != nil {
 		return fmt.Errorf("error fetching pvc %q with %w", pvcName, err)
 	}
@@ -120,10 +117,7 @@ func resizePVCAndValidateSize(pvcPath, appPath string, f *framework.Framework) e
 	opt := metav1.ListOptions{
 		LabelSelector: "app=resize-pvc",
 	}
-
-	pvc, err = f.ClientSet.CoreV1().
-		PersistentVolumeClaims(pvc.Namespace).
-		Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+	pvc, err = getPersistentVolumeClaim(f.ClientSet, pvc.Namespace, pvc.Name)
 	if err != nil {
 		return fmt.Errorf("failed to get pvc: %w", err)
 	}

e2e/snapshot.go

@@ -206,6 +206,71 @@ func createCephFSSnapshotClass(f *framework.Framework) error {
 	return err
 }
 
+func createNFSSnapshotClass(f *framework.Framework) error {
+	scPath := fmt.Sprintf("%s/%s", nfsExamplePath, "snapshotclass.yaml")
+	sc := getSnapshotClass(scPath)
+	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
+	sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName
+
+	fsID, err := getClusterID(f)
+	if err != nil {
+		return fmt.Errorf("failed to get clusterID: %w", err)
+	}
+	sc.Parameters["clusterID"] = fsID
+	sclient, err := newSnapshotClient()
+	if err != nil {
+		return err
+	}
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		_, err = sclient.VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
+		if err != nil {
+			e2elog.Logf("error creating SnapshotClass %q: %v", sc.Name, err)
+			if apierrs.IsAlreadyExists(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to create SnapshotClass %q: %w", sc.Name, err)
+		}
+
+		return true, nil
+	})
+}
+
+func deleteNFSSnapshotClass() error {
+	scPath := fmt.Sprintf("%s/%s", nfsExamplePath, "snapshotclass.yaml")
+	sc := getSnapshotClass(scPath)
+
+	sclient, err := newSnapshotClient()
+	if err != nil {
+		return err
+	}
+
+	timeout := time.Duration(deployTimeout) * time.Minute
+
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		err = sclient.VolumeSnapshotClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{})
+		if err != nil {
+			e2elog.Logf("error deleting SnapshotClass %q: %v", sc.Name, err)
+			if apierrs.IsNotFound(err) {
+				return true, nil
+			}
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to delete SnapshotClass %q: %w", sc.Name, err)
+		}
+
+		return true, nil
+	})
+}
+
 func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSnapshotContent, error) {
 	sclient, err := newSnapshotClient()
 	if err != nil {
@@ -229,6 +294,7 @@ func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSn
 	return volumeSnapshotContent, nil
 }
 
+// nolint:gocyclo,cyclop // reduce complexity
 func validateBiggerPVCFromSnapshot(f *framework.Framework,
 	pvcPath,
 	appPath,
@@ -305,6 +371,43 @@ func validateBiggerPVCFromSnapshot(f *framework.Framework,
 		if err != nil {
 			return fmt.Errorf("failed to validate device size: %w", err)
 		}
+
+		// make sure we had unset snapshot metadata on CreateVolume
+		// from snapshot
+		var (
+			volSnapName        string
+			volSnapNamespace   string
+			volSnapContentName string
+			stdErr             string
+			imageList          []string
+		)
+		imageList, err = listRBDImages(f, defaultRBDPool)
+		if err != nil {
+			e2elog.Failf("failed to list rbd images: %v", err)
+		}
+		e2elog.Logf("list of rbd images: %v", imageList)
+		volSnapName, stdErr, err = execCommandInToolBoxPod(f,
+			formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapNameKey),
+			rookNamespace)
+		if checkGetKeyError(err, stdErr) {
+			e2elog.Failf("found volume snapshot name %s/%s %s=%s: err=%v stdErr=%q",
+				rbdOptions(defaultRBDPool), imageList[0], volSnapNameKey, volSnapName, err, stdErr)
+		}
+		volSnapNamespace, stdErr, err = execCommandInToolBoxPod(f,
+			formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapNamespaceKey),
+			rookNamespace)
+		if checkGetKeyError(err, stdErr) {
+			e2elog.Failf("found volume snapshot namespace %s/%s %s=%s: err=%v stdErr=%q",
+				rbdOptions(defaultRBDPool), imageList[0], volSnapNamespaceKey, volSnapNamespace, err, stdErr)
+		}
+		volSnapContentName, stdErr, err = execCommandInToolBoxPod(f,
+			formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapContentNameKey),
+			rookNamespace)
+		if checkGetKeyError(err, stdErr) {
+			e2elog.Failf("found snapshotcontent name %s/%s %s=%s: err=%v stdErr=%q",
+				rbdOptions(defaultRBDPool), imageList[0], volSnapContentNameKey,
+				volSnapContentName, err, stdErr)
+		}
 	}
 	err = deletePVCAndApp("", f, pvcClone, appClone)
 	if err != nil {

e2e/upgrade-cephfs.go

@@ -390,9 +390,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
 				opt := metav1.ListOptions{
 					LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
 				}
-				pvc, err = f.ClientSet.CoreV1().
-					PersistentVolumeClaims(pvc.Namespace).
-					Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+				pvc, err = getPersistentVolumeClaim(f.ClientSet, pvc.Namespace, pvc.Name)
 				if err != nil {
 					e2elog.Failf("failed to get pvc: %v", err)
 				}

e2e/upgrade-rbd.go

@@ -401,9 +401,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
 					LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
 				}
 				var err error
-				pvc, err = f.ClientSet.CoreV1().
-					PersistentVolumeClaims(pvc.Namespace).
-					Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+				pvc, err = getPersistentVolumeClaim(f.ClientSet, pvc.Namespace, pvc.Name)
 				if err != nil {
 					e2elog.Failf("failed to get pvc: %v", err)
 				}

e2e/utils.go

@@ -49,6 +49,12 @@ const (
 	defaultNs     = "default"
 	defaultSCName = ""
 
+	rbdType    = "rbd"
+	cephfsType = "cephfs"
+
+	volumesType = "volumes"
+	snapsType   = "snaps"
+
 	rookToolBoxPodLabel = "app=rook-ceph-tools"
 	rbdMountOptions     = "mountOptions"
 
@@ -62,6 +68,8 @@ const (
 	noError = ""
 	// labels/selector used to list/delete rbd pods.
 	rbdPodLabels = "app in (ceph-csi-rbd, csi-rbdplugin, csi-rbdplugin-provisioner)"
+
+	exitOneErr = "command terminated with exit code 1"
 )
 
 var (
@@ -69,8 +77,10 @@ var (
 	deployTimeout    int
 	deployCephFS     bool
 	deployRBD        bool
+	deployNFS        bool
 	testCephFS       bool
 	testRBD          bool
+	testNFS          bool
 	helmTest         bool
 	upgradeTesting   bool
 	upgradeVersion   string
@@ -80,8 +90,145 @@ var (
 	poll             = 2 * time.Second
 	isOpenShift      bool
 	clusterID        string
+	nfsDriverName    string
 )
 
+type cephfsFilesystem struct {
+	Name         string `json:"name"`
+	MetadataPool string `json:"metadata_pool"`
+}
+
+// listCephFSFileSystems list CephFS filesystems in json format.
+func listCephFSFileSystems(f *framework.Framework) ([]cephfsFilesystem, error) {
+	var fsList []cephfsFilesystem
+
+	stdout, stdErr, err := execCommandInToolBoxPod(
+		f,
+		"ceph fs ls --format=json",
+		rookNamespace)
+	if err != nil {
+		return fsList, err
+	}
+	if stdErr != "" {
+		return fsList, fmt.Errorf("error listing fs %v", stdErr)
+	}
+
+	err = json.Unmarshal([]byte(stdout), &fsList)
+	if err != nil {
+		return fsList, err
+	}
+
+	return fsList, nil
+}
+
+// getCephFSMetadataPoolName get CephFS pool name from filesystem name.
+func getCephFSMetadataPoolName(f *framework.Framework, filesystem string) (string, error) {
+	fsList, err := listCephFSFileSystems(f)
+	if err != nil {
+		return "", fmt.Errorf("list CephFS filesystem failed: %w", err)
+	}
+	for _, v := range fsList {
+		if v.Name != filesystem {
+			continue
+		}
+
+		return v.MetadataPool, nil
+	}
+
+	return "", fmt.Errorf("metadata pool name not found for filesystem: %s", filesystem)
+}
+
+func compareStdoutWithCount(stdOut string, count int) error {
+	stdOut = strings.TrimSuffix(stdOut, "\n")
+	res, err := strconv.Atoi(stdOut)
+	if err != nil {
+		return fmt.Errorf("failed to convert string to int: %v", stdOut)
+	}
+	if res != count {
+		return fmt.Errorf("expected omap object count %d, got %d", count, res)
+	}
+
+	return nil
+}
+
+// validateOmapCount validates no of OMAP entries on the given pool.
+// Works with Cephfs and RBD drivers and mode can be snapsType or volumesType.
+func validateOmapCount(f *framework.Framework, count int, driver, pool, mode string) {
+	type radosListCommand struct {
+		volumeMode                           string
+		driverType                           string
+		radosLsCmd, radosLsCmdFilter         string
+		radosLsKeysCmd, radosLsKeysCmdFilter string
+	}
+
+	radosListCommands := []radosListCommand{
+		{
+			volumeMode: volumesType,
+			driverType: cephfsType,
+			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
+			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.volume.",
+				pool),
+			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi", pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi|wc -l",
+				pool),
+		},
+		{
+			volumeMode:           volumesType,
+			driverType:           rbdType,
+			radosLsCmd:           fmt.Sprintf("rados ls %s", rbdOptions(pool)),
+			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.volume.", rbdOptions(pool)),
+			radosLsKeysCmd:       fmt.Sprintf("rados listomapkeys csi.volumes.default %s", rbdOptions(pool)),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default %s | wc -l", rbdOptions(pool)),
+		},
+		{
+			volumeMode: snapsType,
+			driverType: cephfsType,
+			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
+			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.snap.",
+				pool),
+			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi", pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi|wc -l",
+				pool),
+		},
+		{
+			volumeMode:           snapsType,
+			driverType:           rbdType,
+			radosLsCmd:           fmt.Sprintf("rados ls %s", rbdOptions(pool)),
+			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.snap.", rbdOptions(pool)),
+			radosLsKeysCmd:       fmt.Sprintf("rados listomapkeys csi.snaps.default %s", rbdOptions(pool)),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default %s | wc -l", rbdOptions(pool)),
+		},
+	}
+
+	for _, cmds := range radosListCommands {
+		if !strings.EqualFold(cmds.volumeMode, mode) || !strings.EqualFold(cmds.driverType, driver) {
+			continue
+		}
+		filterCmds := []string{cmds.radosLsCmdFilter, cmds.radosLsKeysCmdFilter}
+		filterLessCmds := []string{cmds.radosLsCmd, cmds.radosLsKeysCmd}
+		for i, cmd := range filterCmds {
+			stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+			if err != nil || stdErr != "" {
+				if !strings.Contains(err.Error(), exitOneErr) {
+					e2elog.Failf("failed to execute rados command '%s' : err=%v stdErr=%s", cmd, err, stdErr)
+				}
+			}
+			err = compareStdoutWithCount(stdOut, count)
+			if err == nil {
+				continue
+			}
+			saveErr := err
+			if strings.Contains(err.Error(), "expected omap object count") {
+				stdOut, stdErr, err = execCommandInToolBoxPod(f, filterLessCmds[i], rookNamespace)
+				if err == nil {
+					e2elog.Logf("additional debug info: rados ls command output: %s, stdErr: %s", stdOut, stdErr)
+				}
+			}
+			e2elog.Failf("%v", saveErr)
+		}
+	}
+}
+
 func getMons(ns string, c kubernetes.Interface) ([]string, error) {
 	opt := metav1.ListOptions{
 		LabelSelector: "app=rook-ceph-mon",
@@ -651,6 +798,7 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
 func validatePVCClone(
 	totalCount int,
 	sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath,
+	restoreSCName,
 	dataPool string,
 	kms kmsConfig,
 	validatePVC validateFunc,
@@ -682,9 +830,7 @@ func validatePVCClone(
 	}
 
 	checkSum := ""
-	pvc, err = f.ClientSet.CoreV1().
-		PersistentVolumeClaims(pvc.Namespace).
-		Get(context.TODO(), pvc.Name, metav1.GetOptions{})
+	pvc, err = getPersistentVolumeClaim(f.ClientSet, pvc.Namespace, pvc.Name)
 	if err != nil {
 		e2elog.Failf("failed to get pvc %v", err)
 	}
@@ -702,6 +848,10 @@ func validatePVCClone(
 	}
 	pvcClone.Spec.DataSource.Name = pvc.Name
 	pvcClone.Namespace = f.UniqueName
+	if restoreSCName != "" {
+		pvcClone.Spec.StorageClassName = &restoreSCName
+	}
+
 	appClone, err := loadApp(clonePvcAppPath)
 	if err != nil {
 		e2elog.Failf("failed to load application: %v", err)
@@ -1409,7 +1559,10 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(
@@ -1445,7 +1598,10 @@ func retryKubectlFile(namespace string, action kubectlAction, filename string, t
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(
@@ -1478,7 +1634,10 @@ func retryKubectlArgs(namespace string, action kubectlAction, t int, args ...str
 			if isRetryableAPIError(err) {
 				return false, nil
 			}
-			if isAlreadyExistsCLIError(err) {
+			if action == kubectlCreate && isAlreadyExistsCLIError(err) {
+				return true, nil
+			}
+			if action == kubectlDelete && isNotFoundCLIError(err) {
 				return true, nil
 			}
 			e2elog.Logf(

examples/nfs/pod-clone.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: csi-nfs-clone-demo-app
+spec:
+  containers:
+    - name: web-server
+      image: docker.io/library/nginx:latest
+      volumeMounts:
+        - name: mypvc
+          mountPath: /var/lib/www
+  volumes:
+    - name: mypvc
+      persistentVolumeClaim:
+        claimName: nfs-pvc-clone
+        readOnly: false

examples/nfs/pod-restore.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: csi-nfs-restore-demo-pod
+spec:
+  containers:
+    - name: web-server
+      image: docker.io/library/nginx:latest
+      volumeMounts:
+        - name: mypvc
+          mountPath: /var/lib/www
+  volumes:
+    - name: mypvc
+      persistentVolumeClaim:
+        claimName: nfs-pvc-restore
+        readOnly: false

examples/nfs/pod-rwop.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: csi-nfs-demo-rwop-pod
+spec:
+  containers:
+    - name: web-server
+      image: docker.io/library/nginx:latest
+      volumeMounts:
+        - name: mypvc
+          mountPath: /var/lib/www
+  volumes:
+    - name: mypvc
+      persistentVolumeClaim:
+        claimName: csi-nfs-rwop-pvc
+        readOnly: false

examples/nfs/pvc-clone.yaml

@@ -0,0 +1,15 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nfs-pvc-clone
+spec:
+  storageClassName: csi-nfs-sc
+  dataSource:
+    name: csi-nfs-pvc
+    kind: PersistentVolumeClaim
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi

examples/nfs/pvc-restore.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: nfs-pvc-restore
+spec:
+  storageClassName: csi-nfs-sc
+  dataSource:
+    name: nfs-pvc-snapshot
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi

examples/nfs/pvc-rwop.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: csi-nfs-rwop-pvc
+spec:
+  accessModes:
+    - ReadWriteOncePod
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-nfs-sc

examples/nfs/rook-nfs.yaml

@@ -3,12 +3,12 @@ apiVersion: ceph.rook.io/v1
 kind: CephNFS
 metadata:
   name: my-nfs
-  namespace: default
 spec:
   # For Ceph v15, the rados block is required. It is ignored for Ceph v16.
   rados:
-    # Ceph v16 always uses/expects "nfs-ganesha"
-    pool: nfs-ganesha
+    # some versions of Ceph v16 always uses/expects "nfs-ganesha" more recent
+    # versions always use ".nfs"
+    pool: .nfs
     # RADOS namespace where NFS client recovery data is stored in the pool.
     # fixed value for Ceph v16: the name of this CephNFS object
     namespace: my-nfs

examples/nfs/snapshot.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: nfs-pvc-snapshot
+spec:
+  volumeSnapshotClassName: csi-nfsplugin-snapclass
+  source:
+    persistentVolumeClaimName: csi-nfs-pvc

examples/nfs/snapshotclass.yaml

@@ -0,0 +1,23 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: csi-nfsplugin-snapclass
+driver: nfs.csi.ceph.com
+parameters:
+  # String representing a Ceph cluster to provision storage from.
+  # Should be unique across all Ceph clusters in use for provisioning,
+  # cannot be greater than 36 bytes in length, and should remain immutable for
+  # the lifetime of the StorageClass in use.
+  # Ensure to create an entry in the configmap named ceph-csi-config, based on
+  # csi-config-map-sample.yaml, to accompany the string chosen to
+  # represent the Ceph cluster in clusterID below
+  clusterID: <cluster-id>
+
+  # Prefix to use for naming CephFS snapshots.
+  # If omitted, defaults to "csi-snap-".
+  # snapshotNamePrefix: "foo-bar-"
+
+  csi.storage.k8s.io/snapshotter-secret-name: csi-cephfs-secret
+  csi.storage.k8s.io/snapshotter-secret-namespace: default
+deletionPolicy: Delete

examples/nfs/storageclass.yaml

@@ -46,4 +46,4 @@ parameters:
   volumeNamePrefix: nfs-export-
 
 reclaimPolicy: Delete
-allowVolumeExpansion: false
+allowVolumeExpansion: true

go.mod

@@ -4,8 +4,8 @@ go 1.17
 
 require (
 	github.com/IBM/keyprotect-go-client v0.7.0
-	github.com/aws/aws-sdk-go v1.44.5
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.4
+	github.com/aws/aws-sdk-go v1.44.20
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.5
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	// TODO: API for managing NFS-exports requires `ceph_ci_untested` build-tag
 	github.com/ceph/go-ceph v0.15.0
@@ -22,27 +22,29 @@ require (
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.19.0
 	github.com/pborman/uuid v1.2.1
-	github.com/prometheus/client_golang v1.12.1
+	github.com/prometheus/client_golang v1.12.2
 	github.com/stretchr/testify v1.7.1
-	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
-	google.golang.org/grpc v1.46.0
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158
+	google.golang.org/grpc v1.46.2
 	google.golang.org/protobuf v1.28.0
-	k8s.io/api v0.23.6
-	k8s.io/apimachinery v0.23.6
+	k8s.io/api v0.24.0
+	k8s.io/apimachinery v0.24.0
 	k8s.io/client-go v12.0.0+incompatible
-	k8s.io/cloud-provider v0.23.6
+	k8s.io/cloud-provider v0.24.0
 	k8s.io/klog/v2 v2.60.1
 	//
 	// when updating k8s.io/kubernetes, make sure to update the replace section too
 	//
-	k8s.io/kubernetes v1.23.6
-	k8s.io/mount-utils v0.23.6
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed
+	k8s.io/kubernetes v1.24.0
+	k8s.io/mount-utils v0.24.0
+	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
 	sigs.k8s.io/controller-runtime v0.11.0-beta.0.0.20211208212546-f236f0345ad2
 )
 
 require (
+	github.com/PuerkitoBio/purell v1.1.1 // indirect
+	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect
@@ -51,27 +53,29 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
 	github.com/aws/smithy-go v1.11.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/blang/semver v3.5.1+incompatible // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/cyphar/filepath-securejoin v0.2.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/go-sql-driver/mysql v1.5.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
 	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -95,7 +99,9 @@ require (
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/mattn/go-colorable v0.1.6 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
@@ -107,11 +113,11 @@ require (
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/oklog/run v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/runc v1.0.2 // indirect
-	github.com/opencontainers/selinux v1.8.2 // indirect
+	github.com/opencontainers/selinux v1.10.0 // indirect
 	github.com/openshift/api v0.0.0-20210927171657-636513e97fda // indirect
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -120,7 +126,7 @@ require (
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
-	github.com/spf13/cobra v1.2.1 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.opentelemetry.io/contrib v0.20.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
@@ -134,26 +140,28 @@ require (
 	go.opentelemetry.io/proto/otlp v0.7.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 // indirect
+	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/apiserver v0.23.6 // indirect
-	k8s.io/component-base v0.23.6 // indirect
-	k8s.io/component-helpers v0.23.6 // indirect
-	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/apiextensions-apiserver v0.23.0 // indirect
+	k8s.io/apiserver v0.24.0 // indirect
+	k8s.io/component-base v0.24.0 // indirect
+	k8s.io/component-helpers v0.24.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
 	k8s.io/kubectl v0.0.0 // indirect
 	k8s.io/kubelet v0.0.0 // indirect
+	k8s.io/pod-security-admission v0.0.0 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
@@ -167,31 +175,31 @@ replace (
 	//
 	// k8s.io/kubernetes depends on these k8s.io packages, but unversioned
 	//
-	k8s.io/api => k8s.io/api v0.23.6
-	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.23.6
-	k8s.io/apimachinery => k8s.io/apimachinery v0.23.6
-	k8s.io/apiserver => k8s.io/apiserver v0.23.6
-	k8s.io/cli-runtime => k8s.io/cli-runtime v0.23.6
-	k8s.io/client-go => k8s.io/client-go v0.23.6
-	k8s.io/cloud-provider => k8s.io/cloud-provider v0.23.6
-	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.6
-	k8s.io/code-generator => k8s.io/code-generator v0.23.6
-	k8s.io/component-base => k8s.io/component-base v0.23.6
-	k8s.io/component-helpers => k8s.io/component-helpers v0.23.6
-	k8s.io/controller-manager => k8s.io/controller-manager v0.23.6
-	k8s.io/cri-api => k8s.io/cri-api v0.23.6
-	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.23.6
-	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.23.6
-	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.23.6
-	k8s.io/kube-proxy => k8s.io/kube-proxy v0.23.6
-	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.23.6
-	k8s.io/kubectl => k8s.io/kubectl v0.23.6
-	k8s.io/kubelet => k8s.io/kubelet v0.23.6
-	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.23.6
-	k8s.io/metrics => k8s.io/metrics v0.23.6
-	k8s.io/mount-utils => k8s.io/mount-utils v0.23.6
-	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.23.6
-	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.23.6
+	k8s.io/api => k8s.io/api v0.24.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.24.0
+	k8s.io/apiserver => k8s.io/apiserver v0.24.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.24.0
+	k8s.io/client-go => k8s.io/client-go v0.24.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.24.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.0
+	k8s.io/code-generator => k8s.io/code-generator v0.24.0
+	k8s.io/component-base => k8s.io/component-base v0.24.0
+	k8s.io/component-helpers => k8s.io/component-helpers v0.24.0
+	k8s.io/controller-manager => k8s.io/controller-manager v0.24.0
+	k8s.io/cri-api => k8s.io/cri-api v0.24.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.24.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.24.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.24.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.24.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.24.0
+	k8s.io/kubectl => k8s.io/kubectl v0.24.0
+	k8s.io/kubelet => k8s.io/kubelet v0.24.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.24.0
+	k8s.io/metrics => k8s.io/metrics v0.24.0
+	k8s.io/mount-utils => k8s.io/mount-utils v0.24.0
+	k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.24.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.24.0
 	// layeh.com seems to be misbehaving
 	layeh.com/radius => github.com/layeh/radius v0.0.0-20190322222518-890bc1058917
 )

go.sum

@@ -46,7 +46,6 @@ git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqbl
 github.com/Azure/azure-sdk-for-go v36.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/azure-sdk-for-go v55.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
-github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
@@ -96,7 +95,9 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo=
 github.com/Sectorbob/mlab-ns2 v0.0.0-20171030222938-d3aa0c295a8a/go.mod h1:D73UAuEPckrDorYZdtlCu2ySOLuPB5W4rhIkmmc/XbI=
@@ -131,6 +132,8 @@ github.com/armon/go-proxyproto v0.0.0-20190211145416-68259f75880e/go.mod h1:QmP9
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM=
@@ -138,8 +141,8 @@ github.com/aws/aws-sdk-go v1.25.37/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/aws/aws-sdk-go v1.25.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
 github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
-github.com/aws/aws-sdk-go v1.44.5 h1:T4mckpWUfplPG4GA3FDWDCM1QaCzisjGzzeCVBhHKwQ=
-github.com/aws/aws-sdk-go v1.44.5/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.20 h1:nllTRN24EfhDSeKsNbIc6HoC8Ogd2NCJTRB8l84kDlM=
+github.com/aws/aws-sdk-go v1.44.20/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go-v2 v1.16.3 h1:0W1TSJ7O6OzwuEvIXAtJGvOeQ0SGAhcpxPN2/NK5EhM=
 github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 h1:uFWgo6mGJI1n17nbcvSc6fxVuR3xLNqvXt12JCnEcT8=
@@ -148,8 +151,8 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 h1:cnsvEKSoHN4oAN7spMM
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4/go.mod h1:8glyUqVIM4AmeenIsPo0oVh3+NUwnsQml2OFupfQW+0=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 h1:b16QW0XWl0jWjLABFc1A+uh145Oqv+xDcObNk0iQgUk=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4/go.mod h1:uKkN7qmSIsNJVyMtxNQoCEYMvFEXbOg9fwCJPdfp2u8=
-github.com/aws/aws-sdk-go-v2/service/sts v1.16.4 h1:+xtV90n3abQmgzk1pS++FdxZTrPEDgQng6e4/56WR2A=
-github.com/aws/aws-sdk-go-v2/service/sts v1.16.4/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE=
+github.com/aws/aws-sdk-go-v2/service/sts v1.16.5 h1:aEMP+STBsvCY1U/yL4pAqUrU4oFli46AdbCXvVbTiHI=
+github.com/aws/aws-sdk-go-v2/service/sts v1.16.5/go.mod h1:lfSYenAXtavyX2A1LsViglqlG9eEFYxNryTZS5rn3QE=
 github.com/aws/smithy-go v1.11.2 h1:eG/N+CcUMAvsdffgMvjMKwfyDzIkjM6pfxMJ8Mzc6mE=
 github.com/aws/smithy-go v1.11.2/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
@@ -163,13 +166,12 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
-github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA=
-github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
-github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
+github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -190,13 +192,13 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
-github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
+github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
 github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0/go.mod h1:5d8DqS60xkj9k3aXfL3+mXBH0DPYO0FQjcKosxl+b/Q=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
-github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -225,15 +227,15 @@ github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1
 github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw=
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
+github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.11/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.12/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/corefile-migration v1.0.14/go.mod h1:XnhgULOEouimnzgn0t4WPuFDN2/PJQcTxdWKC5eXNGE=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -252,6 +254,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -260,8 +263,7 @@ github.com/csi-addons/replication-lib-utils v0.2.0/go.mod h1:ROQlEsc2EerVtc/K/C+
 github.com/csi-addons/spec v0.1.0/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
 github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe h1:Q2sxgtdRV4Je1R2eLCUPrR/KQZxkSbesGrpCjl0/mU4=
 github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
-github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
-github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU=
 github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ=
 github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg=
@@ -276,9 +278,10 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
-github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
+github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -300,6 +303,7 @@ github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjl
 github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
 github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -377,10 +381,13 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE
 github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
 github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng=
 github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -396,6 +403,7 @@ github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaL
 github.com/gocql/gocql v0.0.0-20190402132108-0e1d5de854df/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -426,6 +434,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -439,9 +448,11 @@ github.com/gomodules/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DK
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
-github.com/google/cadvisor v0.43.0/go.mod h1:+RdMSbc3FVr5NYCD2dOEJy/LI0jYJ/0xJXkzWXEyiFQ=
-github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
+github.com/google/cadvisor v0.44.1/go.mod h1:GQ9KQfz0iNHQk3D6ftzJWK4TXabfIgM10Oy3FkR+Gzg=
+github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w=
 github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA=
+github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54=
+github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -489,7 +500,6 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
 github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
-github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
 github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
 github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
@@ -705,6 +715,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -770,6 +781,7 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -822,9 +834,9 @@ github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
 github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ=
 github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
-github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
-github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
-github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A=
+github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
+github.com/moby/sys/mountinfo v0.6.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
+github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -839,6 +851,7 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
 github.com/mwielbut/pointy v1.1.0/go.mod h1:MvvO+uMFj9T5DMda33HlvogsFBX7pWWKAkFIn4teYwY=
@@ -884,14 +897,15 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg=
-github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
+github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
+github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc=
-github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU=
+github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
 github.com/openshift/api v0.0.0-20210105115604-44119421ec6b/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg=
 github.com/openshift/api v0.0.0-20210927171657-636513e97fda h1:VoJmrqbFDuqzjlByItbjx/HxmReK4LC+X3Jt2Wv2Ogs=
 github.com/openshift/api v0.0.0-20210927171657-636513e97fda/go.mod h1:RsQCVJu4qhUawxxDP7pGlwU3IA4F01wYm3qKEu29Su8=
@@ -953,8 +967,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
 github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
 github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
+github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
+github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -997,6 +1012,7 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
@@ -1004,7 +1020,7 @@ github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIH
 github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v2.19.9+incompatible h1:IrPVlK4nfwW10DF7pW+7YJKws9NkgNzWozwwWv9FsgY=
@@ -1037,8 +1053,9 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
 github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
+github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -1093,16 +1110,19 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
 go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ=
 go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0=
+go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q=
 go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE=
 go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc=
 go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4=
@@ -1178,8 +1198,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE=
+golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1227,6 +1248,7 @@ golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hM
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1281,9 +1303,9 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT
 golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
@@ -1305,8 +1327,9 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw=
 golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg=
+golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1379,9 +1402,7 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1399,19 +1420,22 @@ golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1435,8 +1459,9 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
+golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1501,9 +1526,10 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
+golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1605,8 +1631,9 @@ google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
 google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2 h1:NHN4wOCScVzKhPenJ2dt+BTs3X/XkBVI/Rh4iDt55T8=
 google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 h1:Et6SkiuvnBn+SgrSYXs/BrUpGB4mbdwt4R3vaPIlicA=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
@@ -1638,8 +1665,8 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ
 google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
-google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
@@ -1710,31 +1737,32 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw=
-k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g=
-k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q=
-k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU=
-k8s.io/apimachinery v0.23.6 h1:RH1UweWJkWNTlFx0D8uxOpaU1tjIOvVVWV/bu5b3/NQ=
-k8s.io/apimachinery v0.23.6/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
-k8s.io/apiserver v0.23.6 h1:p94LiXcsSnpSDIl4cv98liBuFKcaygSCNopFNfMg/Ac=
-k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg=
-k8s.io/cli-runtime v0.23.6/go.mod h1:0Z3VR/HRIFKiLzKIAkm1mPtcH98GT/fXu2IU0E4vFmw=
-k8s.io/client-go v0.23.6 h1:7h4SctDVQAQbkHQnR4Kzi7EyUyvla5G1pFWf4+Od7hQ=
-k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4=
-k8s.io/cloud-provider v0.23.6 h1:/dO0LOBiyDBV/d0tu9MjA+YCo4Bti8sy2o2jbAXI01Y=
-k8s.io/cloud-provider v0.23.6/go.mod h1:kk827l5bRFlSBvSWTR0tdNzQqa44gJdqmVRjdnN+X4g=
-k8s.io/cluster-bootstrap v0.23.6/go.mod h1:JbFrtiOjW1YDujFuUzPbuOSCnSUIkXUjEeci5BxB8ug=
-k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
-k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8=
-k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0=
-k8s.io/component-helpers v0.23.6 h1:95O4LX4mk2LHWfdn/5rQJ7hOzd/teKRh9fCNXr2tJic=
-k8s.io/component-helpers v0.23.6/go.mod h1:kgvl6wvnYg9oebklLPpbW8UhvAZ9Qds26/RANEbny/8=
-k8s.io/controller-manager v0.23.6/go.mod h1:szOgeulY0c3xLUa9RWX1uPCjLr76RmVttghr4xQzC5U=
-k8s.io/cri-api v0.23.6/go.mod h1:REJE3PSU0h/LOV1APBrupxrEJqnoxZC8KWzkBUHwrK4=
-k8s.io/csi-translation-lib v0.23.6/go.mod h1:8UtizkEUAnIOuTBko4/mswp3tkfMftdCEzRxJAUfxAw=
+k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg=
+k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I=
+k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY=
+k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM=
+k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ=
+k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM=
+k8s.io/apiserver v0.24.0 h1:GR7kGsjOMfilRvlG3Stxv/3uz/ryvJ/aZXc5pqdsNV0=
+k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA=
+k8s.io/cli-runtime v0.24.0/go.mod h1:9XxoZDsEkRFUThnwqNviqzljtT/LdHtNWvcNFrAXl0A=
+k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U=
+k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw=
+k8s.io/cloud-provider v0.24.0 h1:kQ6zB2oy0VDl+6vdRAKEbtwDM1MmuhNCyA/v+Fk2g30=
+k8s.io/cloud-provider v0.24.0/go.mod h1:cqkEWJWzToaqtS5ti8KQJQcL2IWssWGXHzicxZyaC6s=
+k8s.io/cluster-bootstrap v0.24.0/go.mod h1:xw+IfoaUweMCAoi+VYhmqkcjii2G7gNg59dmGn7hi0g=
+k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w=
+k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g=
+k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA=
+k8s.io/component-helpers v0.24.0 h1:hZIHGfdd55thhqd9oxjDTw68OAPauDMJ+8hC69aNw1I=
+k8s.io/component-helpers v0.24.0/go.mod h1:Q2SlLm4h6g6lPTC9GMMfzdywfLSvJT2f1hOnnjaWD8c=
+k8s.io/controller-manager v0.24.0/go.mod h1:ageMNQZc7cNH0FF1oarm7wZs6XyJj/V82nNVmgPaeDU=
+k8s.io/cri-api v0.24.0/go.mod h1:t3tImFtGeStN+ES69bQUX9sFg67ek38BM9YIJhMmuig=
+k8s.io/csi-translation-lib v0.24.0/go.mod h1:jJaC3a1tI3IShByiAQmOOCl5PKpiZ51Vh70c9Eg2msM=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
+k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/klog v0.3.0 h1:0VPpR+sizsiivjIfIAQH/rl8tan6jvWkS7lU+0di3lE=
 k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
@@ -1744,33 +1772,35 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc=
 k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
-k8s.io/kube-aggregator v0.23.6/go.mod h1:cubFdoSJRMEN+ilg1ErhNIoplJwyYbmgn3bUlen8KjA=
-k8s.io/kube-controller-manager v0.23.6/go.mod h1:7QZIxuo2Y1XSsCuFf+2d7EN458xWCC+0Q6f5hT0puAc=
+k8s.io/kube-aggregator v0.24.0/go.mod h1:ftfs6Fi46z3cKzeF2kvNBPLbMlSKuqZbesJGNp/cQnw=
+k8s.io/kube-controller-manager v0.24.0/go.mod h1:s0pbwI8UuBEDdXQbTUpQdNIyU4rQ7jOxaXAcRBoWpJQ=
 k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
 k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
-k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
-k8s.io/kube-proxy v0.23.6/go.mod h1:HyB7KVZ88iRtKkrr3mFPIL+k5tZZObk/2MjHl41Po68=
-k8s.io/kube-scheduler v0.23.6/go.mod h1:BqgJohUEgt2Fsg9cBipnyDiaPgexodxivrbNEACcsqQ=
-k8s.io/kubectl v0.23.6 h1:ajzrqj88GqlH/gpscMCts+mKNeSJprpkWJEHO8CR2Ss=
-k8s.io/kubectl v0.23.6/go.mod h1:mMtJhc2QtQiSfvIQoMEUIjGHtZuP4uxMy/ees6j6zx8=
-k8s.io/kubelet v0.23.6 h1:tuscMqYCt9cxWursmTU9OJ2tPLv66Ji+AGbuV1Z/lug=
-k8s.io/kubelet v0.23.6/go.mod h1:ROttmKIUkB9in4NyX/SfnAoXGfW/Dju3VCGFP34F5ac=
-k8s.io/kubernetes v1.23.6 h1:IGoo8aUBuIep0G4NW5ezejuH16sMX0zd2sx/bD+D+Z0=
-k8s.io/kubernetes v1.23.6/go.mod h1:avI3LUTUYZugxwh52KMVM7v9ZjB5gYJ6D3FIoZ1SHUo=
-k8s.io/legacy-cloud-providers v0.23.6/go.mod h1:jc5asdak2B4UW77dOWTTK8XvJak+MxyXdCHr4r/M7mA=
-k8s.io/metrics v0.23.6/go.mod h1:Fm9VzVMZ7KVEEeLStF2y3XogfcDwpGyI15o1xB6PbYk=
-k8s.io/mount-utils v0.23.6 h1:NTLirnhIWsv/UtBYLNpwtftBPeoAAwPocyx0B2KsCxs=
-k8s.io/mount-utils v0.23.6/go.mod h1:OTN3LQPiOGMfx/SmVlsnySwsAmh4gYrDYLchlMHtf98=
-k8s.io/pod-security-admission v0.23.6/go.mod h1:molyEVwa8EtBxQ/LgizUHnN8l0su2PQtLSROnF3P9Js=
-k8s.io/sample-apiserver v0.23.6/go.mod h1:32pvunKq7Px1NZsIxGwQKy8o+i4FfK/2rCL/htr7Uig=
-k8s.io/system-validators v1.6.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
+k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 h1:Gii5eqf+GmIEwGNKQYQClCayuJCe2/4fZUvF7VG99sU=
+k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk=
+k8s.io/kube-proxy v0.24.0/go.mod h1:OZ1k9jSwW94Rmj5hepCFea7qlGvvU+bfcosc6+dcFKA=
+k8s.io/kube-scheduler v0.24.0/go.mod h1:DUq+fXaC51N1kl2YnT2EZSxOph6JOmIJe/pQe5keZPc=
+k8s.io/kubectl v0.24.0 h1:nA+WtMLVdXUs4wLogGd1mPTAesnLdBpCVgCmz3I7dXo=
+k8s.io/kubectl v0.24.0/go.mod h1:pdXkmCyHiRTqjYfyUJiXtbVNURhv0/Q1TyRhy2d5ic0=
+k8s.io/kubelet v0.24.0 h1:fH+D6mSr4DGIeHp/O2+mCEJhkVq3Gpgv9BVOHI+GrWY=
+k8s.io/kubelet v0.24.0/go.mod h1:p3BBacmHTCMpUf+nluhlyzuGHmONKAspqCvpu9oPAyA=
+k8s.io/kubernetes v1.24.0 h1:9qRjlCuMjooyFTXLxduMBT+MZSdROWa3idI1AXZirVs=
+k8s.io/kubernetes v1.24.0/go.mod h1:8e8maMiZzBR2/8Po5Uulx+MXZUYJuN3vtKwD4Ct1Xi0=
+k8s.io/legacy-cloud-providers v0.24.0/go.mod h1:j2gujMUYBEtbYfJaL8JUOgInzERm9fxJwEaOkZcnEUk=
+k8s.io/metrics v0.24.0/go.mod h1:jrLlFGdKl3X+szubOXPG0Lf2aVxuV3QJcbsgVRAM6fI=
+k8s.io/mount-utils v0.24.0 h1:1SCkAY99QUchRa00HkLcm0HXajy8xlWHvue4wYdvBVU=
+k8s.io/mount-utils v0.24.0/go.mod h1:XrSqB3a2e8sq+aU+rlbcBtQ3EgcuDk5RP9ZsGxjoDrI=
+k8s.io/pod-security-admission v0.24.0 h1:nTZtZPdJ5ZusFyuxGZxfGxQ5piuhJyxuG5YmVUWG/Gs=
+k8s.io/pod-security-admission v0.24.0/go.mod h1:YBS4mAdoba2qMvLPE3S7eMIxGlqUf4amHH26jUUqXX4=
+k8s.io/sample-apiserver v0.24.0/go.mod h1:6YGSatoHMHIac/2dTtARwYH8PVWY5qq1L9ZYbxZ9lHY=
+k8s.io/system-validators v1.7.0/go.mod h1:gP1Ky+R9wtrSiFbrpEPwWMeYz9yqyy1S/KOh0Vci7WI=
 k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
-k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE=
 k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc=
+k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
 modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=
@@ -1785,12 +1815,13 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lR
 sigs.k8s.io/controller-runtime v0.2.2/go.mod h1:9dyohw3ZtoXQuV1e766PHUn+cmrRCIcBh6XIMFNMZ+I=
 sigs.k8s.io/controller-runtime v0.11.0-beta.0.0.20211208212546-f236f0345ad2 h1:+ReKrjTrd57mtAU19BJkxSAaWRIQkFlaWcO6dGFVP1g=
 sigs.k8s.io/controller-runtime v0.11.0-beta.0.0.20211208212546-f236f0345ad2/go.mod h1:KKwLiTooNGu+JmLZGn9Sl3Gjmfj66eMbCQznLP5zcqA=
-sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s=
 sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs=
-sigs.k8s.io/kustomize/api v0.10.1/go.mod h1:2FigT1QN6xKdcnGS2Ppp1uIWrtWN28Ms8A3OZUZhwr8=
-sigs.k8s.io/kustomize/cmd/config v0.10.2/go.mod h1:K2aW7nXJ0AaT+VA/eO0/dzFLxmpFcTzudmAgDwPY1HQ=
-sigs.k8s.io/kustomize/kustomize/v4 v4.4.1/go.mod h1:qOKJMMz2mBP+vcS7vK+mNz4HBLjaQSWRY22EF6Tb7Io=
-sigs.k8s.io/kustomize/kyaml v0.13.0/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E=
+sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y=
+sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=
+sigs.k8s.io/kustomize/api v0.11.4/go.mod h1:k+8RsqYbgpkIrJ4p9jcdPqe8DprLxFUUO0yNOq8C+xI=
+sigs.k8s.io/kustomize/cmd/config v0.10.6/go.mod h1:/S4A4nUANUa4bZJ/Edt7ZQTyKOY9WCER0uBS1SW2Rco=
+sigs.k8s.io/kustomize/kustomize/v4 v4.5.4/go.mod h1:Zo/Xc5FKD6sHl0lilbrieeGeZHVYCA4BzxeAaLI05Bg=
+sigs.k8s.io/kustomize/kyaml v0.13.6/go.mod h1:yHP031rn1QX1lr/Xd934Ri/xdVNG8BE2ECa78Ht/kEg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.0/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y=

internal/kms/vault_sa.go

@@ -208,7 +208,7 @@ func (kms *vaultTenantSA) parseConfig(config map[string]interface{}) error {
 	} else if err == nil {
 		kms.vaultConfig[vault.AuthMountPath], err = detectAuthMountPath(vaultAuthPath)
 		if err != nil {
-			return fmt.Errorf("failed to set %s in Vault config: %w", vault.AuthMountPath, err)
+			return fmt.Errorf("failed to set \"vaultAuthPath\" in Vault config: %w", err)
 		}
 	}
 

internal/nfs/controller/controllerserver.go

@@ -46,6 +46,7 @@ type Server struct {
 func NewControllerServer(d *csicommon.CSIDriver) *Server {
 	// global instance of the volume journal, yuck
 	store.VolJournal = journal.NewCSIVolumeJournalWithNamespace(cephfs.CSIInstanceID, fsutil.RadosNamespace)
+	store.SnapJournal = journal.NewCSISnapshotJournalWithNamespace(cephfs.CSIInstanceID, fsutil.RadosNamespace)
 
 	return &Server{
 		backendServer: cephfs.NewControllerServer(d),
@@ -150,3 +151,28 @@ func (cs *Server) DeleteVolume(
 
 	return cs.backendServer.DeleteVolume(ctx, req)
 }
+
+// ControllerExpandVolume calls the backend (CephFS) procedure to expand the
+// volume. There is no interaction with the NFS-server needed to publish the
+// new size.
+func (cs *Server) ControllerExpandVolume(
+	ctx context.Context,
+	req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) {
+	return cs.backendServer.ControllerExpandVolume(ctx, req)
+}
+
+// CreateSnapshot calls the backend (CephFS) procedure to create snapshot.
+// There is no interaction with the NFS-server needed for snapshot creation.
+func (cs *Server) CreateSnapshot(
+	ctx context.Context,
+	req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
+	return cs.backendServer.CreateSnapshot(ctx, req)
+}
+
+// DeleteSnapshot calls the backend (CephFS) procedure to delete snapshot.
+// There is no interaction with the NFS-server needed for snapshot creation.
+func (cs *Server) DeleteSnapshot(
+	ctx context.Context,
+	req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) {
+	return cs.backendServer.DeleteSnapshot(ctx, req)
+}

internal/nfs/driver/driver.go

@@ -46,6 +46,9 @@ func (fs *Driver) Run(conf *util.Config) {
 	cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
 		csi.ControllerServiceCapability_RPC_SINGLE_NODE_MULTI_WRITER,
+		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
+		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+		csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
 	})
 	// VolumeCapabilities are validated by the CephFS Controller
 	cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{

internal/rbd/clone.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	librbd "github.com/ceph/go-ceph/rbd"
@@ -144,11 +145,9 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol
 		return err
 	}
 
-	if parentVol.isEncrypted() {
-		err = parentVol.copyEncryptionConfig(&rv.rbdImage, false)
-		if err != nil {
-			return fmt.Errorf("failed to copy encryption config for %q: %w", rv, err)
-		}
+	err = parentVol.copyEncryptionConfig(&rv.rbdImage, true)
+	if err != nil {
+		return fmt.Errorf("failed to copy encryption config for %q: %w", rv, err)
 	}
 
 	err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID)
@@ -208,6 +207,13 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro
 		}
 	}()
 
+	err = tempClone.unsetAllMetadata(k8s.GetVolumeMetadataKeys())
+	if err != nil {
+		log.ErrorLog(ctx, "failed to unset volume metadata on temp clone image %q: %v", tempClone, err)
+
+		return err
+	}
+
 	// create snap of temp clone from temporary cloned image
 	// create final clone
 	// delete snap of temp clone
@@ -216,5 +222,10 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro
 		return errClone
 	}
 
+	err = parentVol.copyEncryptionConfig(&rv.rbdImage, true)
+	if err != nil {
+		return fmt.Errorf("failed to copy encryption config for %q: %w", rv, err)
+	}
+
 	return nil
 }

internal/rbd/controllerserver.go

@@ -19,6 +19,7 @@ package rbd
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
 	"github.com/ceph/ceph-csi/internal/util"
@@ -330,7 +331,8 @@ func (cs *ControllerServer) CreateVolume(
 	}
 
 	// Set Metadata on PV Create
-	err = rbdVol.setVolumeMetadata(req.GetParameters())
+	metadata := k8s.GetVolumeMetadata(req.GetParameters())
+	err = rbdVol.setAllMetadata(metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -453,7 +455,8 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
 	}
 
 	// Set metadata on restart of provisioner pod when image exist
-	err := rbdVol.setVolumeMetadata(req.GetParameters())
+	metadata := k8s.GetVolumeMetadata(req.GetParameters())
+	err := rbdVol.setAllMetadata(metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -588,9 +591,20 @@ func (cs *ControllerServer) createVolumeFromSnapshot(
 
 		return err
 	}
+	err = rbdVol.unsetAllMetadata(k8s.GetSnapshotMetadataKeys())
+	if err != nil {
+		log.ErrorLog(ctx, "failed to unset snapshot metadata on rbd image %q: %v", rbdVol, err)
+
+		return err
+	}
 
 	log.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol, rbdSnap)
 
+	err = parentVol.copyEncryptionConfig(&rbdVol.rbdImage, true)
+	if err != nil {
+		return fmt.Errorf("failed to copy encryption config for %q: %w", rbdVol, err)
+	}
+
 	// resize the volume if the size is different
 	// expand the image if the requested size is greater than the current size
 	err = rbdVol.expand()
@@ -963,7 +977,7 @@ func (cs *ControllerServer) ValidateVolumeCapabilities(
 }
 
 // CreateSnapshot creates the snapshot in backend and stores metadata in store.
-// nolint:cyclop // TODO: reduce complexity
+// nolint:gocyclo,cyclop // TODO: reduce complexity.
 func (cs *ControllerServer) CreateSnapshot(
 	ctx context.Context,
 	req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) {
@@ -1066,9 +1080,14 @@ func (cs *ControllerServer) CreateSnapshot(
 	// Update the metadata on snapshot not on the original image
 	rbdVol.RbdImageName = rbdSnap.RbdSnapName
 
+	err = rbdVol.unsetAllMetadata(k8s.GetVolumeMetadataKeys())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
 	// Set snapshot-name/snapshot-namespace/snapshotcontent-name details
 	// on RBD backend image as metadata on create
-	err = rbdVol.setSnapshotMetadata(req.GetParameters())
+	metadata := k8s.GetSnapshotMetadata(req.GetParameters())
+	err = rbdVol.setAllMetadata(metadata)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
 	}
@@ -1104,11 +1123,9 @@ func cloneFromSnapshot(
 	}
 	defer vol.Destroy()
 
-	if rbdVol.isEncrypted() {
-		err = rbdVol.copyEncryptionConfig(&vol.rbdImage, false)
-		if err != nil {
-			return nil, status.Error(codes.Internal, err.Error())
-		}
+	err = rbdVol.copyEncryptionConfig(&vol.rbdImage, false)
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
 	}
 
 	err = vol.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
@@ -1127,7 +1144,8 @@ func cloneFromSnapshot(
 	// Update snapshot-name/snapshot-namespace/snapshotcontent-name details on
 	// RBD backend image as metadata on restart of provisioner pod when image exist
 	if len(parameters) != 0 {
-		err = rbdVol.setSnapshotMetadata(parameters)
+		metadata := k8s.GetSnapshotMetadata(parameters)
+		err = rbdVol.setAllMetadata(metadata)
 		if err != nil {
 			return nil, status.Error(codes.Internal, err.Error())
 		}
@@ -1207,14 +1225,12 @@ func (cs *ControllerServer) doSnapshotClone(
 		}
 	}()
 
-	if parentVol.isEncrypted() {
-		cryptErr := parentVol.copyEncryptionConfig(&cloneRbd.rbdImage, false)
-		if cryptErr != nil {
-			log.WarningLog(ctx, "failed copy encryption "+
-				"config for %q: %v", cloneRbd, cryptErr)
+	err = parentVol.copyEncryptionConfig(&cloneRbd.rbdImage, false)
+	if err != nil {
+		log.ErrorLog(ctx, "failed to copy encryption "+
+			"config for %q: %v", cloneRbd, err)
 
-			return nil, err
-		}
+		return nil, err
 	}
 
 	err = cloneRbd.createSnapshot(ctx, rbdSnap)

internal/rbd/driver/driver.go

@@ -17,7 +17,9 @@ limitations under the License.
 package rbddriver
 
 import (
+	"errors"
 	"fmt"
+	"os"
 
 	casrbd "github.com/ceph/ceph-csi/internal/csi-addons/rbd"
 	csiaddons "github.com/ceph/ceph-csi/internal/csi-addons/server"
@@ -144,7 +146,7 @@ func (r *Driver) Run(conf *util.Config) {
 		}
 		var attr string
 		attr, err = rbd.GetKrbdSupportedFeatures()
-		if err != nil {
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
 			log.FatalLogMsg(err.Error())
 		}
 		var krbdFeatures uint

internal/rbd/encryption.go

@@ -120,14 +120,20 @@ func (ri *rbdImage) setupEncryption(ctx context.Context) error {
 }
 
 // copyEncryptionConfig copies the VolumeEncryption object from the source
-// rbdImage to the passed argument. This function re-encrypts the passphrase
-// from the original, so that both encrypted passphrases (potentially, depends
-// on the DEKStore) have different contents.
+// rbdImage to the passed argument if the source rbdImage is encrypted.
+// This function re-encrypts the passphrase  from the original, so that
+// both encrypted passphrases (potentially, depends on the DEKStore) have
+// different contents.
 // When copyOnlyPassphrase is set to true, only the passphrase is copied to the
 // destination rbdImage's VolumeEncryption object which needs to be initialized
 // beforehand and is possibly different from the source VolumeEncryption
 // (Usecase: Restoring snapshot into a storageclass with different encryption config).
 func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) error {
+	// nothing to do if parent image is not encrypted.
+	if !ri.isEncrypted() {
+		return nil
+	}
+
 	if ri.VolID == cp.VolID {
 		return fmt.Errorf("BUG: %q and %q have the same VolID (%s) "+
 			"set!? Call stack: %s", ri, cp, ri.VolID, util.CallStack())
@@ -184,7 +190,7 @@ func (ri *rbdImage) repairEncryptionConfig(dest *rbdImage) error {
 			dest.conn = ri.conn.Copy()
 		}
 
-		return ri.copyEncryptionConfig(dest, false)
+		return ri.copyEncryptionConfig(dest, true)
 	}
 
 	return nil

internal/rbd/nodeserver.go

@@ -144,6 +144,7 @@ func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *r
 // this function also receive the credentials and secrets args as it differs in its data.
 // The credentials are used directly by functions like voljournal.Connect() and other functions
 // like genVolFromVolumeOptions() make use of secrets.
+// nolint:gocyclo,cyclop // reduce complexity
 func populateRbdVol(
 	ctx context.Context,
 	req *csi.NodeStageVolumeRequest,
@@ -241,8 +242,15 @@ func populateRbdVol(
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	if req.GetVolumeContext()["mounter"] == rbdDefaultMounter &&
-		!isKrbdFeatureSupported(ctx, strings.Join(rv.ImageFeatureSet.Names(), ",")) {
+	features := strings.Join(rv.ImageFeatureSet.Names(), ",")
+	isFeatureExist, err := isKrbdFeatureSupported(ctx, features)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		log.ErrorLog(ctx, "failed checking krbd features %q: %v", features, err)
+
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	if rv.Mounter == rbdDefaultMounter && !isFeatureExist {
 		if !parseBoolOption(ctx, req.GetVolumeContext(), tryOtherMounters, false) {
 			log.ErrorLog(ctx, "unsupported krbd Feature, set `tryOtherMounters:true` or fix krbd driver")
 			err = errors.New("unsupported krbd Feature")
@@ -251,8 +259,6 @@ func populateRbdVol(
 		}
 		// fallback to rbd-nbd,
 		rv.Mounter = rbdNbdMounter
-	} else {
-		rv.Mounter = req.GetVolumeContext()["mounter"]
 	}
 
 	err = getMapOptions(req, rv)

internal/rbd/rbd_journal.go

@@ -324,8 +324,8 @@ func (rv *rbdVolume) Exists(ctx context.Context, parentVol *rbdVolume) (bool, er
 		return false, err
 	}
 
-	if parentVol != nil && parentVol.isEncrypted() {
-		err = parentVol.copyEncryptionConfig(&rv.rbdImage, false)
+	if parentVol != nil {
+		err = parentVol.copyEncryptionConfig(&rv.rbdImage, true)
 		if err != nil {
 			log.ErrorLog(ctx, err.Error())
 
@@ -617,7 +617,7 @@ func RegenerateJournal(
 		}
 		// Update Metadata on reattach of the same old PV
 		parameters := k8s.PrepareVolumeMetadata(claimName, rbdVol.Owner, "")
-		err = rbdVol.setVolumeMetadata(parameters)
+		err = rbdVol.setAllMetadata(parameters)
 		if err != nil {
 			return "", fmt.Errorf("failed to set volume metadata: %w", err)
 		}

internal/rbd/rbd_util.go

@@ -28,7 +28,6 @@ import (
 	"time"
 
 	"github.com/ceph/ceph-csi/internal/util"
-	"github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 
 	"github.com/ceph/go-ceph/rados"
@@ -193,28 +192,83 @@ type migrationVolID struct {
 	clusterID string
 }
 
-var supportedFeatures = map[string]imageFeature{
-	librbd.FeatureNameLayering: {
-		needRbdNbd: false,
-	},
-	librbd.FeatureNameExclusiveLock: {
-		needRbdNbd: false,
-	},
-	librbd.FeatureNameObjectMap: {
-		needRbdNbd: false,
-		dependsOn:  []string{librbd.FeatureNameExclusiveLock},
-	},
-	librbd.FeatureNameFastDiff: {
-		needRbdNbd: false,
-		dependsOn:  []string{librbd.FeatureNameObjectMap},
-	},
-	librbd.FeatureNameJournaling: {
-		needRbdNbd: true,
-		dependsOn:  []string{librbd.FeatureNameExclusiveLock},
-	},
-	librbd.FeatureNameDeepFlatten: {
-		needRbdNbd: false,
-	},
+var (
+	supportedFeatures = map[string]imageFeature{
+		librbd.FeatureNameLayering: {
+			needRbdNbd: false,
+		},
+		librbd.FeatureNameExclusiveLock: {
+			needRbdNbd: false,
+		},
+		librbd.FeatureNameObjectMap: {
+			needRbdNbd: false,
+			dependsOn:  []string{librbd.FeatureNameExclusiveLock},
+		},
+		librbd.FeatureNameFastDiff: {
+			needRbdNbd: false,
+			dependsOn:  []string{librbd.FeatureNameObjectMap},
+		},
+		librbd.FeatureNameJournaling: {
+			needRbdNbd: true,
+			dependsOn:  []string{librbd.FeatureNameExclusiveLock},
+		},
+		librbd.FeatureNameDeepFlatten: {
+			needRbdNbd: false,
+		},
+	}
+
+	krbdLayeringSupport = []util.KernelVersion{
+		{
+			Version:    3,
+			PatchLevel: 8,
+			SubLevel:   0,
+		},
+	}
+	krbdStripingV2Support = []util.KernelVersion{
+		{
+			Version:    3,
+			PatchLevel: 10,
+			SubLevel:   0,
+		},
+	}
+	krbdExclusiveLockSupport = []util.KernelVersion{
+		{
+			Version:    4,
+			PatchLevel: 9,
+			SubLevel:   0,
+		},
+	}
+	krbdDataPoolSupport = []util.KernelVersion{
+		{
+			Version:    4,
+			PatchLevel: 11,
+			SubLevel:   0,
+		},
+	}
+)
+
+// prepareKrbdFeatureAttrs prepare krbd fearure set based on kernel version.
+// Minimum kernel version should be 3.8, else it will return error.
+func prepareKrbdFeatureAttrs() (uint64, error) {
+	// fetch the current running kernel info
+	release, err := util.GetKernelVersion()
+	if err != nil {
+		return 0, fmt.Errorf("fetching current kernel version failed: %w", err)
+	}
+
+	switch {
+	case util.CheckKernelSupport(release, krbdDataPoolSupport):
+		return librbd.FeatureDataPool, nil
+	case util.CheckKernelSupport(release, krbdExclusiveLockSupport):
+		return librbd.FeatureExclusiveLock, nil
+	case util.CheckKernelSupport(release, krbdStripingV2Support):
+		return librbd.FeatureStripingV2, nil
+	case util.CheckKernelSupport(release, krbdLayeringSupport):
+		return librbd.FeatureLayering, nil
+	}
+	log.ErrorLogMsg("kernel version is too old: %q", release)
+
+	return 0, os.ErrNotExist
 }
 
 // GetKrbdSupportedFeatures load the module if needed and return supported
@@ -239,9 +293,19 @@ func GetKrbdSupportedFeatures() (string, error) {
 	}
 	val, err := os.ReadFile(krbdSupportedFeaturesFile)
 	if err != nil {
-		log.ErrorLogMsg("reading file %q failed: %v", krbdSupportedFeaturesFile, err)
+		if !errors.Is(err, os.ErrNotExist) {
+			log.ErrorLogMsg("reading file %q failed: %v", krbdSupportedFeaturesFile, err)
 
-		return "", err
+			return "", err
+		}
+		attrs, err := prepareKrbdFeatureAttrs()
+		if err != nil {
+			log.ErrorLogMsg("preparing krbd feature attributes failed, %v", err)
+
+			return "", err
+		}
+
+		return strconv.FormatUint(attrs, 16), nil
 	}
 
 	return strings.TrimSuffix(string(val), "\n"), nil
@@ -264,7 +328,12 @@ func HexStringToInteger(hexString string) (uint, error) {
 
 // isKrbdFeatureSupported checks if a given Image Feature is supported by krbd
 // driver or not.
-func isKrbdFeatureSupported(ctx context.Context, imageFeatures string) bool {
+func isKrbdFeatureSupported(ctx context.Context, imageFeatures string) (bool, error) {
+	// return false when /sys/bus/rbd/supported_features is absent and we are
+	// not in a position to prepare krbd feature attributes, i.e. if kernel <= 3.8
+	if krbdFeatures == 0 {
+		return false, os.ErrNotExist
+	}
 	arr := strings.Split(imageFeatures, ",")
 	log.UsefulLog(ctx, "checking for ImageFeatures: %v", arr)
 	imageFeatureSet := librbd.FeatureSetFromNames(arr)
@@ -279,7 +348,7 @@ func isKrbdFeatureSupported(ctx context.Context, imageFeatures string) bool {
 		}
 	}
 
-	return supported
+	return supported, nil
 }
 
 // Connect an rbdVolume to the Ceph cluster.
@@ -1366,15 +1435,6 @@ func (rv *rbdVolume) cloneRbdImageFromSnapshot(
 		}
 	}()
 
-	if pSnapOpts.isEncrypted() {
-		pSnapOpts.conn = rv.conn.Copy()
-
-		err = pSnapOpts.copyEncryptionConfig(&rv.rbdImage, true)
-		if err != nil {
-			return fmt.Errorf("failed to clone encryption config: %w", err)
-		}
-	}
-
 	// get image latest information
 	err = rv.getImageInfo()
 	if err != nil {
@@ -1922,9 +1982,9 @@ func genVolFromVolIDWithMigration(
 	return rv, err
 }
 
-// setVolumeMetadata set PV/PVC/PVCNamespace metadata on RBD image.
-func (rv *rbdVolume) setVolumeMetadata(parameters map[string]string) error {
-	for k, v := range k8s.GetVolumeMetadata(parameters) {
+// setAllMetadata set all the metadata from arg parameters on RBD image.
+func (rv *rbdVolume) setAllMetadata(parameters map[string]string) error {
+	for k, v := range parameters {
 		err := rv.SetMetadata(k, v)
 		if err != nil {
 			return fmt.Errorf("failed to set metadata key %q, value %q on image: %w", k, v, err)
@@ -1934,13 +1994,13 @@ func (rv *rbdVolume) setVolumeMetadata(parameters map[string]string) error {
 	return nil
 }
 
-// setSnapshotMetadata Set snapshot-name/snapshot-namespace/snapshotcontent-name metadata
-// on RBD image.
-func (rv *rbdVolume) setSnapshotMetadata(parameters map[string]string) error {
-	for k, v := range k8s.GetSnapshotMetadata(parameters) {
-		err := rv.SetMetadata(k, v)
-		if err != nil {
-			return fmt.Errorf("failed to set metadata key %q, value %q on image: %w", k, v, err)
+// unsetAllMetadata unset all the metadata from arg keys on RBD image.
+func (rv *rbdVolume) unsetAllMetadata(keys []string) error {
+	for _, key := range keys {
+		err := rv.RemoveMetadata(key)
+		// TODO: replace string comparison with errno.
+		if err != nil && !strings.Contains(err.Error(), "No such file or directory") {
+			return fmt.Errorf("failed to unset metadata key %q on %q: %w", key, rv, err)
 		}
 	}
 

internal/rbd/rbd_util_test.go

@@ -18,6 +18,7 @@ package rbd
 
 import (
 	"context"
+	"errors"
 	"io/ioutil"
 	"os"
 	"strings"
@@ -346,8 +347,13 @@ func TestIsKrbdFeatureSupported(t *testing.T) {
 			if err != nil {
 				t.Errorf("HexStringToInteger(%s) failed", krbdSupportedFeaturesAttr)
 			}
-			supported := isKrbdFeatureSupported(ctx, tc.featureName)
-			if supported != tc.isSupported {
+			// In case /sys/bus/rbd/supported_features is absent and we are
+			// not in a position to prepare krbd feature attributes,
+			// isKrbdFeatureSupported returns error ErrNotExist
+			supported, err := isKrbdFeatureSupported(ctx, tc.featureName)
+			if err != nil && !errors.Is(err, os.ErrNotExist) {
+				t.Errorf("isKrbdFeatureSupported(%s) returned error: %v", tc.featureName, err)
+			} else if supported != tc.isSupported {
 				t.Errorf("isKrbdFeatureSupported(%s) returned supported status, expected: %t, got: %t",
 					tc.featureName, tc.isSupported, supported)
 			}

internal/util/k8s/parameters.go

@@ -69,6 +69,15 @@ func GetVolumeMetadata(parameters map[string]string) map[string]string {
 	return newParam
 }
 
+// GetVolumeMetadataKeys return volume metadata keys.
+func GetVolumeMetadataKeys() []string {
+	return []string{
+		pvcNameKey,
+		pvcNamespaceKey,
+		pvNameKey,
+	}
+}
+
 // PrepareVolumeMetadata return PV/PVC/PVCNamespace metadata based on inputs.
 func PrepareVolumeMetadata(pvcName, pvcNamespace, pvName string) map[string]string {
 	newParam := map[string]string{}
@@ -100,3 +109,12 @@ func GetSnapshotMetadata(parameters map[string]string) map[string]string {
 
 	return newParam
 }
+
+// GetSnapshotMetadataKeys return snapshot metadata keys.
+func GetSnapshotMetadataKeys() []string {
+	return []string{
+		volSnapNameKey,
+		volSnapNamespaceKey,
+		volSnapContentNameKey,
+	}
+}

internal/util/pidlimit.go

@@ -27,16 +27,19 @@ import (
 )
 
 const (
-	procCgroup    = "/proc/self/cgroup"
-	sysPidsMaxFmt = "/sys/fs/cgroup/pids%s/pids.max"
+	procCgroup            = "/proc/self/cgroup"
+	sysPidsMaxFmtCgroupV1 = "/sys/fs/cgroup/pids%s/pids.max"
+	sysPidsMaxFmtCgroupV2 = "/sys/fs/cgroup%s/pids.max"
 )
 
-// return the cgouprs "pids.max" file of the current process
-//
-// find the line containing the pids group from the /proc/self/cgroup file
-// $ grep 'pids' /proc/self/cgroup
+// getCgroupPidsFile return the cgroups "pids.max" file of the
+// current process
+// For cgroup v1, find the line containing the pids group from the /proc/self/cgroup file
+// $ grep ':pids:' /proc/self/cgroup
 // 7:pids:/kubepods.slice/kubepods-besteffort.slice/....scope
 // $ cat /sys/fs/cgroup/pids + *.scope + /pids.max.
+// The entry for cgroup v2 is always in the format "0::...scope", no subsystem given.
+// (see https://www.kernel.org/doc/Documentation/cgroup-v2.txt)
 func getCgroupPidsFile() (string, error) {
 	cgroup, err := os.Open(procCgroup)
 	if err != nil {
@@ -44,6 +47,7 @@ func getCgroupPidsFile() (string, error) {
 	}
 	defer cgroup.Close() // #nosec: error on close is not critical here
 
+	pidsMax := ""
 	scanner := bufio.NewScanner(cgroup)
 	var slice string
 	for scanner.Scan() {
@@ -51,8 +55,16 @@ func getCgroupPidsFile() (string, error) {
 		if parts == nil || len(parts) < 3 {
 			continue
 		}
+		// No cgroup subsystem given, then it is cgroupv2
+		if parts[0] == "0" && parts[1] == "" {
+			slice = parts[2]
+			pidsMax = fmt.Sprintf(sysPidsMaxFmtCgroupV2, slice)
+
+			break
+		}
 		if parts[1] == "pids" {
 			slice = parts[2]
+			pidsMax = fmt.Sprintf(sysPidsMaxFmtCgroupV1, slice)
 
 			break
 		}
@@ -61,8 +73,6 @@ func getCgroupPidsFile() (string, error) {
 		return "", fmt.Errorf("could not find a cgroup for 'pids'")
 	}
 
-	pidsMax := fmt.Sprintf(sysPidsMaxFmt, slice)
-
 	return pidsMax, nil
 }
 

scripts/Dockerfile.test

@@ -18,6 +18,7 @@ ENV \
  GOPATH=${GOPATH} \
  GOROOT=${GOROOT} \
  GO111MODULE=on \
+ CEPHCSIPATH=/go/src/github.com/ceph/ceph-csi \
  PATH="${GOPATH}/bin:${GOROOT}/bin:/opt/commitlint/node_modules/.bin:${PATH}"
 
 COPY build.env /
@@ -53,6 +54,7 @@ RUN source /build.env \
     && npm init -y \
     && npm install @commitlint/cli@"${COMMITLINT_VERSION}" \
     && popd \
+    && git config --global --add safe.directory ${CEPHCSIPATH} \
     && true
 
-WORKDIR /go/src/github.com/ceph/ceph-csi
+WORKDIR ${CEPHCSIPATH}

scripts/minikube.sh

@@ -227,8 +227,10 @@ up)
     KUBE_MAJOR=$(kube_version 1)
     KUBE_MINOR=$(kube_version 2)
     if [ "${KUBE_MAJOR}" -eq 1 ] && [ "${KUBE_MINOR}" -ge 22 ];then
-        # if kubernetes version is greater than 1.22 enable RWOP feature gate
-        K8S_FEATURE_GATES="${K8S_FEATURE_GATES},ReadWriteOncePod=true"
+        K8S_FEATURE_GATES="${K8S_FEATURE_GATES},ReadWriteOncePod=true,PodSecurity=false"
+    fi
+    if [ "${KUBE_MAJOR}" -eq 1 ] && [ "${KUBE_MINOR}" -ge 23 ];then
+        K8S_FEATURE_GATES="${K8S_FEATURE_GATES},RecoverVolumeExpansionFailure=true"
     fi
     # shellcheck disable=SC2086
     ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP} --wait-timeout="${MINIKUBE_WAIT_TIMEOUT}" --wait="${MINIKUBE_WAIT}" --delete-on-failure ${DISK_CONFIG}

vendor/github.com/PuerkitoBio/purell/.gitignore

@@ -0,0 +1,5 @@
+*.sublime-*
+.DS_Store
+*.swp
+*.swo
+tags

vendor/github.com/PuerkitoBio/purell/.travis.yml

@@ -0,0 +1,12 @@
+language: go
+
+go:
+    - 1.4.x
+    - 1.5.x
+    - 1.6.x
+    - 1.7.x
+    - 1.8.x
+    - 1.9.x
+    - "1.10.x"
+    - "1.11.x"
+    - tip

vendor/github.com/PuerkitoBio/purell/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2012, Martin Angers
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/PuerkitoBio/purell/README.md

@@ -0,0 +1,188 @@
+# Purell
+
+Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know...
+
+Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc].
+
+[![build status](https://travis-ci.org/PuerkitoBio/purell.svg?branch=master)](http://travis-ci.org/PuerkitoBio/purell)
+
+## Install
+
+`go get github.com/PuerkitoBio/purell`
+
+## Changelog
+
+*    **v1.1.1** : Fix failing test due to Go1.12 changes (thanks to @ianlancetaylor).
+*    **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121).
+*    **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich).
+*    **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]).
+*    **v0.2.0** : Add benchmarks, Attempt IDN support.
+*    **v0.1.0** : Initial release.
+
+## Examples
+
+From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."):
+
+```go
+package purell
+
+import (
+  "fmt"
+  "net/url"
+)
+
+func ExampleNormalizeURLString() {
+  if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/",
+    FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil {
+    panic(err)
+  } else {
+    fmt.Print(normalized)
+  }
+  // Output: http://somewebsite.com:80/Amazing%3F/url/
+}
+
+func ExampleMustNormalizeURLString() {
+  normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/",
+    FlagsUnsafeGreedy)
+  fmt.Print(normalized)
+
+  // Output: http://somewebsite.com/Amazing%FA/url
+}
+
+func ExampleNormalizeURL() {
+  if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil {
+    panic(err)
+  } else {
+    normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment)
+    fmt.Print(normalized)
+  }
+
+  // Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0
+}
+```
+
+## API
+
+As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags:
+
+```go
+const (
+	// Safe normalizations
+	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
+	FlagLowercaseHost                                            // http://HOST -> http://host
+	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
+	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
+	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
+	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
+	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
+
+	// Usually safe normalizations
+	FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
+	FlagAddTrailingSlash    // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
+	FlagRemoveDotSegments   // http://host/path/./a/b/../c -> http://host/path/a/c
+
+	// Unsafe normalizations
+	FlagRemoveDirectoryIndex   // http://host/path/index.html -> http://host/path/
+	FlagRemoveFragment         // http://host/path#fragment -> http://host/path
+	FlagForceHTTP              // https://host -> http://host
+	FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
+	FlagRemoveWWW              // http://www.host/ -> http://host/
+	FlagAddWWW                 // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
+	FlagSortQuery              // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
+
+	// Normalizations not in the wikipedia article, required to cover tests cases
+	// submitted by jehiah
+	FlagDecodeDWORDHost           // http://1113982867 -> http://66.102.7.147
+	FlagDecodeOctalHost           // http://0102.0146.07.0223 -> http://66.102.7.147
+	FlagDecodeHexHost             // http://0x42660793 -> http://66.102.7.147
+	FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
+	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
+
+	// Convenience set of safe normalizations
+	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
+
+	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
+	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
+
+	// Convenience set of usually safe normalizations (includes FlagsSafe)
+	FlagsUsuallySafeGreedy    NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
+	FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
+
+	// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
+	FlagsUnsafeGreedy    NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
+	FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
+
+	// Convenience set of all available flags
+	FlagsAllGreedy    = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+	FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+)
+```
+
+For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set.
+
+The [full godoc reference is available on gopkgdoc][godoc].
+
+Some things to note:
+
+*    `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it.
+
+*    The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*):
+    -    %24 -> $
+    -    %26 -> &
+    -    %2B-%3B -> +,-./0123456789:;
+    -    %3D -> =
+    -    %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ
+    -    %5F -> _
+    -    %61-%7A -> abcdefghijklmnopqrstuvwxyz
+    -    %7E -> ~
+
+
+*    When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization).
+
+*    The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell.
+
+*    The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object.
+
+### Safe vs Usually Safe vs Unsafe
+
+Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between.
+
+Consider the following URL:
+
+`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
+
+Normalizing with the `FlagsSafe` gives:
+
+`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
+
+With the `FlagsUsuallySafeGreedy`:
+
+`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid`
+
+And with `FlagsUnsafeGreedy`:
+
+`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3`
+
+## TODOs
+
+*    Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`.
+
+## Thanks / Contributions
+
+@rogpeppe
+@jehiah
+@opennota
+@pchristopher1275
+@zenovich
+@beeker1121
+
+## License
+
+The [BSD 3-Clause license][bsd].
+
+[bsd]: http://opensource.org/licenses/BSD-3-Clause
+[wiki]: http://en.wikipedia.org/wiki/URL_normalization
+[rfc]: http://tools.ietf.org/html/rfc3986#section-6
+[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell
+[pr5]: https://github.com/PuerkitoBio/purell/pull/5
+[iss7]: https://github.com/PuerkitoBio/purell/issues/7

vendor/github.com/PuerkitoBio/purell/purell.go

@@ -0,0 +1,379 @@
+/*
+Package purell offers URL normalization as described on the wikipedia page:
+http://en.wikipedia.org/wiki/URL_normalization
+*/
+package purell
+
+import (
+	"bytes"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/urlesc"
+	"golang.org/x/net/idna"
+	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/width"
+)
+
+// A set of normalization flags determines how a URL will
+// be normalized.
+type NormalizationFlags uint
+
+const (
+	// Safe normalizations
+	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
+	FlagLowercaseHost                                            // http://HOST -> http://host
+	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
+	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
+	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
+	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
+	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
+
+	// Usually safe normalizations
+	FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
+	FlagAddTrailingSlash    // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
+	FlagRemoveDotSegments   // http://host/path/./a/b/../c -> http://host/path/a/c
+
+	// Unsafe normalizations
+	FlagRemoveDirectoryIndex   // http://host/path/index.html -> http://host/path/
+	FlagRemoveFragment         // http://host/path#fragment -> http://host/path
+	FlagForceHTTP              // https://host -> http://host
+	FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
+	FlagRemoveWWW              // http://www.host/ -> http://host/
+	FlagAddWWW                 // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
+	FlagSortQuery              // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
+
+	// Normalizations not in the wikipedia article, required to cover tests cases
+	// submitted by jehiah
+	FlagDecodeDWORDHost           // http://1113982867 -> http://66.102.7.147
+	FlagDecodeOctalHost           // http://0102.0146.07.0223 -> http://66.102.7.147
+	FlagDecodeHexHost             // http://0x42660793 -> http://66.102.7.147
+	FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
+	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
+
+	// Convenience set of safe normalizations
+	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
+
+	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
+	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
+
+	// Convenience set of usually safe normalizations (includes FlagsSafe)
+	FlagsUsuallySafeGreedy    NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
+	FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
+
+	// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
+	FlagsUnsafeGreedy    NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
+	FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
+
+	// Convenience set of all available flags
+	FlagsAllGreedy    = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+	FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
+)
+
+const (
+	defaultHttpPort  = ":80"
+	defaultHttpsPort = ":443"
+)
+
+// Regular expressions used by the normalizations
+var rxPort = regexp.MustCompile(`(:\d+)/?$`)
+var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
+var rxDupSlashes = regexp.MustCompile(`/{2,}`)
+var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
+var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
+var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
+var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
+var rxEmptyPort = regexp.MustCompile(`:+$`)
+
+// Map of flags to implementation function.
+// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
+// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
+
+// Since maps have undefined traversing order, make a slice of ordered keys
+var flagsOrder = []NormalizationFlags{
+	FlagLowercaseScheme,
+	FlagLowercaseHost,
+	FlagRemoveDefaultPort,
+	FlagRemoveDirectoryIndex,
+	FlagRemoveDotSegments,
+	FlagRemoveFragment,
+	FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
+	FlagRemoveDuplicateSlashes,
+	FlagRemoveWWW,
+	FlagAddWWW,
+	FlagSortQuery,
+	FlagDecodeDWORDHost,
+	FlagDecodeOctalHost,
+	FlagDecodeHexHost,
+	FlagRemoveUnnecessaryHostDots,
+	FlagRemoveEmptyPortSeparator,
+	FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
+	FlagAddTrailingSlash,
+}
+
+// ... and then the map, where order is unimportant
+var flags = map[NormalizationFlags]func(*url.URL){
+	FlagLowercaseScheme:           lowercaseScheme,
+	FlagLowercaseHost:             lowercaseHost,
+	FlagRemoveDefaultPort:         removeDefaultPort,
+	FlagRemoveDirectoryIndex:      removeDirectoryIndex,
+	FlagRemoveDotSegments:         removeDotSegments,
+	FlagRemoveFragment:            removeFragment,
+	FlagForceHTTP:                 forceHTTP,
+	FlagRemoveDuplicateSlashes:    removeDuplicateSlashes,
+	FlagRemoveWWW:                 removeWWW,
+	FlagAddWWW:                    addWWW,
+	FlagSortQuery:                 sortQuery,
+	FlagDecodeDWORDHost:           decodeDWORDHost,
+	FlagDecodeOctalHost:           decodeOctalHost,
+	FlagDecodeHexHost:             decodeHexHost,
+	FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
+	FlagRemoveEmptyPortSeparator:  removeEmptyPortSeparator,
+	FlagRemoveTrailingSlash:       removeTrailingSlash,
+	FlagAddTrailingSlash:          addTrailingSlash,
+}
+
+// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
+// It takes an URL string as input, as well as the normalization flags.
+func MustNormalizeURLString(u string, f NormalizationFlags) string {
+	result, e := NormalizeURLString(u, f)
+	if e != nil {
+		panic(e)
+	}
+	return result
+}
+
+// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
+// It takes an URL string as input, as well as the normalization flags.
+func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
+	parsed, err := url.Parse(u)
+	if err != nil {
+		return "", err
+	}
+
+	if f&FlagLowercaseHost == FlagLowercaseHost {
+		parsed.Host = strings.ToLower(parsed.Host)
+	}
+
+	// The idna package doesn't fully conform to RFC 5895
+	// (https://tools.ietf.org/html/rfc5895), so we do it here.
+	// Taken from Go 1.8 cycle source, courtesy of bradfitz.
+	// TODO: Remove when (if?) idna package conforms to RFC 5895.
+	parsed.Host = width.Fold.String(parsed.Host)
+	parsed.Host = norm.NFC.String(parsed.Host)
+	if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
+		return "", err
+	}
+
+	return NormalizeURL(parsed, f), nil
+}
+
+// NormalizeURL returns the normalized string.
+// It takes a parsed URL object as input, as well as the normalization flags.
+func NormalizeURL(u *url.URL, f NormalizationFlags) string {
+	for _, k := range flagsOrder {
+		if f&k == k {
+			flags[k](u)
+		}
+	}
+	return urlesc.Escape(u)
+}
+
+func lowercaseScheme(u *url.URL) {
+	if len(u.Scheme) > 0 {
+		u.Scheme = strings.ToLower(u.Scheme)
+	}
+}
+
+func lowercaseHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		u.Host = strings.ToLower(u.Host)
+	}
+}
+
+func removeDefaultPort(u *url.URL) {
+	if len(u.Host) > 0 {
+		scheme := strings.ToLower(u.Scheme)
+		u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
+			if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
+				return ""
+			}
+			return val
+		})
+	}
+}
+
+func removeTrailingSlash(u *url.URL) {
+	if l := len(u.Path); l > 0 {
+		if strings.HasSuffix(u.Path, "/") {
+			u.Path = u.Path[:l-1]
+		}
+	} else if l = len(u.Host); l > 0 {
+		if strings.HasSuffix(u.Host, "/") {
+			u.Host = u.Host[:l-1]
+		}
+	}
+}
+
+func addTrailingSlash(u *url.URL) {
+	if l := len(u.Path); l > 0 {
+		if !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+	} else if l = len(u.Host); l > 0 {
+		if !strings.HasSuffix(u.Host, "/") {
+			u.Host += "/"
+		}
+	}
+}
+
+func removeDotSegments(u *url.URL) {
+	if len(u.Path) > 0 {
+		var dotFree []string
+		var lastIsDot bool
+
+		sections := strings.Split(u.Path, "/")
+		for _, s := range sections {
+			if s == ".." {
+				if len(dotFree) > 0 {
+					dotFree = dotFree[:len(dotFree)-1]
+				}
+			} else if s != "." {
+				dotFree = append(dotFree, s)
+			}
+			lastIsDot = (s == "." || s == "..")
+		}
+		// Special case if host does not end with / and new path does not begin with /
+		u.Path = strings.Join(dotFree, "/")
+		if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
+			u.Path = "/" + u.Path
+		}
+		// Special case if the last segment was a dot, make sure the path ends with a slash
+		if lastIsDot && !strings.HasSuffix(u.Path, "/") {
+			u.Path += "/"
+		}
+	}
+}
+
+func removeDirectoryIndex(u *url.URL) {
+	if len(u.Path) > 0 {
+		u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
+	}
+}
+
+func removeFragment(u *url.URL) {
+	u.Fragment = ""
+}
+
+func forceHTTP(u *url.URL) {
+	if strings.ToLower(u.Scheme) == "https" {
+		u.Scheme = "http"
+	}
+}
+
+func removeDuplicateSlashes(u *url.URL) {
+	if len(u.Path) > 0 {
+		u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
+	}
+}
+
+func removeWWW(u *url.URL) {
+	if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+		u.Host = u.Host[4:]
+	}
+}
+
+func addWWW(u *url.URL) {
+	if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
+		u.Host = "www." + u.Host
+	}
+}
+
+func sortQuery(u *url.URL) {
+	q := u.Query()
+
+	if len(q) > 0 {
+		arKeys := make([]string, len(q))
+		i := 0
+		for k := range q {
+			arKeys[i] = k
+			i++
+		}
+		sort.Strings(arKeys)
+		buf := new(bytes.Buffer)
+		for _, k := range arKeys {
+			sort.Strings(q[k])
+			for _, v := range q[k] {
+				if buf.Len() > 0 {
+					buf.WriteRune('&')
+				}
+				buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
+			}
+		}
+
+		// Rebuild the raw query string
+		u.RawQuery = buf.String()
+	}
+}
+
+func decodeDWORDHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+			var parts [4]int64
+
+			dword, _ := strconv.ParseInt(matches[1], 10, 0)
+			for i, shift := range []uint{24, 16, 8, 0} {
+				parts[i] = dword >> shift & 0xFF
+			}
+			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
+		}
+	}
+}
+
+func decodeOctalHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
+			var parts [4]int64
+
+			for i := 1; i <= 4; i++ {
+				parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
+			}
+			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
+		}
+	}
+}
+
+func decodeHexHost(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
+			// Conversion is safe because of regex validation
+			parsed, _ := strconv.ParseInt(matches[1], 16, 0)
+			// Set host as DWORD (base 10) encoded host
+			u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
+			// The rest is the same as decoding a DWORD host
+			decodeDWORDHost(u)
+		}
+	}
+}
+
+func removeUnncessaryHostDots(u *url.URL) {
+	if len(u.Host) > 0 {
+		if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
+			// Trim the leading and trailing dots
+			u.Host = strings.Trim(matches[1], ".")
+			if len(matches) > 2 {
+				u.Host += matches[2]
+			}
+		}
+	}
+}
+
+func removeEmptyPortSeparator(u *url.URL) {
+	if len(u.Host) > 0 {
+		u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
+	}
+}

vendor/github.com/PuerkitoBio/urlesc/.travis.yml

@@ -0,0 +1,15 @@
+language: go
+
+go:
+  - 1.4.x
+  - 1.5.x
+  - 1.6.x
+  - 1.7.x
+  - 1.8.x
+  - tip
+
+install:
+  - go build .
+
+script:
+  - go test -v

vendor/github.com/PuerkitoBio/urlesc/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Will Fitzgerald. All rights reserved.
+Copyright (c) 2012 The Go Authors. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are

vendor/github.com/PuerkitoBio/urlesc/README.md

@@ -0,0 +1,16 @@
+urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc)
+======
+
+Package urlesc implements query escaping as per RFC 3986.
+
+It contains some parts of the net/url package, modified so as to allow
+some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)).
+
+## Install
+
+    go get github.com/PuerkitoBio/urlesc
+
+## License
+
+Go license (BSD-3-Clause)
+

vendor/github.com/PuerkitoBio/urlesc/urlesc.go

@@ -0,0 +1,180 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package urlesc implements query escaping as per RFC 3986.
+// It contains some parts of the net/url package, modified so as to allow
+// some reserved characters incorrectly escaped by net/url.
+// See https://github.com/golang/go/issues/5684
+package urlesc
+
+import (
+	"bytes"
+	"net/url"
+	"strings"
+)
+
+type encoding int
+
+const (
+	encodePath encoding = 1 + iota
+	encodeUserPassword
+	encodeQueryComponent
+	encodeFragment
+)
+
+// Return true if the specified character should be escaped when
+// appearing in a URL string, according to RFC 3986.
+func shouldEscape(c byte, mode encoding) bool {
+	// §2.3 Unreserved characters (alphanum)
+	if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
+		return false
+	}
+
+	switch c {
+	case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
+		return false
+
+	// §2.2 Reserved characters (reserved)
+	case ':', '/', '?', '#', '[', ']', '@', // gen-delims
+		'!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
+		// Different sections of the URL allow a few of
+		// the reserved characters to appear unescaped.
+		switch mode {
+		case encodePath: // §3.3
+			// The RFC allows sub-delims and : @.
+			// '/', '[' and ']' can be used to assign meaning to individual path
+			// segments.  This package only manipulates the path as a whole,
+			// so we allow those as well.  That leaves only ? and # to escape.
+			return c == '?' || c == '#'
+
+		case encodeUserPassword: // §3.2.1
+			// The RFC allows : and sub-delims in
+			// userinfo.  The parsing of userinfo treats ':' as special so we must escape
+			// all the gen-delims.
+			return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
+
+		case encodeQueryComponent: // §3.4
+			// The RFC allows / and ?.
+			return c != '/' && c != '?'
+
+		case encodeFragment: // §4.1
+			// The RFC text is silent but the grammar allows
+			// everything, so escape nothing but #
+			return c == '#'
+		}
+	}
+
+	// Everything else must be escaped.
+	return true
+}
+
+// QueryEscape escapes the string so it can be safely placed
+// inside a URL query.
+func QueryEscape(s string) string {
+	return escape(s, encodeQueryComponent)
+}
+
+func escape(s string, mode encoding) string {
+	spaceCount, hexCount := 0, 0
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if shouldEscape(c, mode) {
+			if c == ' ' && mode == encodeQueryComponent {
+				spaceCount++
+			} else {
+				hexCount++
+			}
+		}
+	}
+
+	if spaceCount == 0 && hexCount == 0 {
+		return s
+	}
+
+	t := make([]byte, len(s)+2*hexCount)
+	j := 0
+	for i := 0; i < len(s); i++ {
+		switch c := s[i]; {
+		case c == ' ' && mode == encodeQueryComponent:
+			t[j] = '+'
+			j++
+		case shouldEscape(c, mode):
+			t[j] = '%'
+			t[j+1] = "0123456789ABCDEF"[c>>4]
+			t[j+2] = "0123456789ABCDEF"[c&15]
+			j += 3
+		default:
+			t[j] = s[i]
+			j++
+		}
+	}
+	return string(t)
+}
+
+var uiReplacer = strings.NewReplacer(
+	"%21", "!",
+	"%27", "'",
+	"%28", "(",
+	"%29", ")",
+	"%2A", "*",
+)
+
+// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
+func unescapeUserinfo(s string) string {
+	return uiReplacer.Replace(s)
+}
+
+// Escape reassembles the URL into a valid URL string.
+// The general form of the result is one of:
+//
+//	scheme:opaque
+//	scheme://userinfo@host/path?query#fragment
+//
+// If u.Opaque is non-empty, String uses the first form;
+// otherwise it uses the second form.
+//
+// In the second form, the following rules apply:
+//	- if u.Scheme is empty, scheme: is omitted.
+//	- if u.User is nil, userinfo@ is omitted.
+//	- if u.Host is empty, host/ is omitted.
+//	- if u.Scheme and u.Host are empty and u.User is nil,
+//	   the entire scheme://userinfo@host/ is omitted.
+//	- if u.Host is non-empty and u.Path begins with a /,
+//	   the form host/path does not add its own /.
+//	- if u.RawQuery is empty, ?query is omitted.
+//	- if u.Fragment is empty, #fragment is omitted.
+func Escape(u *url.URL) string {
+	var buf bytes.Buffer
+	if u.Scheme != "" {
+		buf.WriteString(u.Scheme)
+		buf.WriteByte(':')
+	}
+	if u.Opaque != "" {
+		buf.WriteString(u.Opaque)
+	} else {
+		if u.Scheme != "" || u.Host != "" || u.User != nil {
+			buf.WriteString("//")
+			if ui := u.User; ui != nil {
+				buf.WriteString(unescapeUserinfo(ui.String()))
+				buf.WriteByte('@')
+			}
+			if h := u.Host; h != "" {
+				buf.WriteString(h)
+			}
+		}
+		if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
+			buf.WriteByte('/')
+		}
+		buf.WriteString(escape(u.Path, encodePath))
+	}
+	if u.RawQuery != "" {
+		buf.WriteByte('?')
+		buf.WriteString(u.RawQuery)
+	}
+	if u.Fragment != "" {
+		buf.WriteByte('#')
+		buf.WriteString(escape(u.Fragment, encodeFragment))
+	}
+	return buf.String()
+}

vendor/github.com/aws/aws-sdk-go-v2/service/sts/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.16.5 (2022-05-16)
+
+* **Documentation**: Documentation updates for AWS Security Token Service.
+
 # v1.16.4 (2022-04-25)
 
 * **Dependency Update**: Updated to the latest SDK module versions

vendor/github.com/aws/aws-sdk-go-v2/service/sts/api_op_GetSessionToken.go

@@ -26,6 +26,11 @@ import (
 // (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html)
 // and Comparing the Amazon Web Services STS API operations
 // (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
+// in the IAM User Guide. No permissions are required for users to perform this
+// operation. The purpose of the sts:GetSessionToken operation is to authenticate
+// the user using MFA. You cannot use policies to control authentication
+// operations. For more information, see Permissions for GetSessionToken
+// (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_getsessiontoken.html)
 // in the IAM User Guide. Session Duration The GetSessionToken operation must be
 // called by using the long-term Amazon Web Services security credentials of the
 // Amazon Web Services account root user or an IAM user. Credentials that are

vendor/github.com/aws/aws-sdk-go-v2/service/sts/go_module_metadata.go

@@ -3,4 +3,4 @@
 package sts
 
 // goModuleVersion is the tagged release for this module
-const goModuleVersion = "1.16.4"
+const goModuleVersion = "1.16.5"

vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go

@@ -548,6 +548,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "ap-southeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-3",
+				}: endpoint{},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{},
@@ -2226,63 +2229,183 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "af-south-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "af-south-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.af-south-1.api.aws",
+				},
 				endpointKey{
 					Region: "ap-east-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-east-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-east-1.api.aws",
+				},
 				endpointKey{
 					Region: "ap-northeast-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-northeast-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-northeast-1.api.aws",
+				},
 				endpointKey{
 					Region: "ap-northeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-northeast-2",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-northeast-2.api.aws",
+				},
 				endpointKey{
 					Region: "ap-south-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-south-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-south-1.api.aws",
+				},
 				endpointKey{
 					Region: "ap-southeast-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-southeast-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-southeast-1.api.aws",
+				},
 				endpointKey{
 					Region: "ap-southeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ap-southeast-2",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ap-southeast-2.api.aws",
+				},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "ca-central-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.ca-central-1.api.aws",
+				},
 				endpointKey{
 					Region: "eu-central-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-central-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-central-1.api.aws",
+				},
 				endpointKey{
 					Region: "eu-north-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-north-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-north-1.api.aws",
+				},
 				endpointKey{
 					Region: "eu-south-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-south-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-south-1.api.aws",
+				},
 				endpointKey{
 					Region: "eu-west-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-west-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-west-1.api.aws",
+				},
 				endpointKey{
 					Region: "eu-west-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-west-2",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-west-2.api.aws",
+				},
 				endpointKey{
 					Region: "eu-west-3",
 				}: endpoint{},
+				endpointKey{
+					Region:  "eu-west-3",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.eu-west-3.api.aws",
+				},
 				endpointKey{
 					Region: "me-south-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "me-south-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.me-south-1.api.aws",
+				},
 				endpointKey{
 					Region: "sa-east-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "sa-east-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.sa-east-1.api.aws",
+				},
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-east-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.us-east-1.api.aws",
+				},
 				endpointKey{
 					Region: "us-east-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-east-2",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.us-east-2.api.aws",
+				},
 				endpointKey{
 					Region: "us-west-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-west-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.us-west-1.api.aws",
+				},
 				endpointKey{
 					Region: "us-west-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-west-2",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.us-west-2.api.aws",
+				},
 			},
 		},
 		"apprunner": service{
@@ -2522,6 +2645,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "eu-west-1",
 				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-2",
+				}: endpoint{},
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
@@ -3523,9 +3649,6 @@ var awsPartition = partition{
 		},
 		"cloudhsm": service{
 			Endpoints: serviceEndpoints{
-				endpointKey{
-					Region: "eu-west-1",
-				}: endpoint{},
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
@@ -6776,7 +6899,7 @@ var awsPartition = partition{
 					Region:  "ap-south-1",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.ap-south-1.aws",
+					Hostname: "ec2.ap-south-1.api.aws",
 				},
 				endpointKey{
 					Region: "ap-southeast-1",
@@ -6812,7 +6935,7 @@ var awsPartition = partition{
 					Region:  "eu-west-1",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.eu-west-1.aws",
+					Hostname: "ec2.eu-west-1.api.aws",
 				},
 				endpointKey{
 					Region: "eu-west-2",
@@ -6875,7 +6998,7 @@ var awsPartition = partition{
 					Region:  "sa-east-1",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.sa-east-1.aws",
+					Hostname: "ec2.sa-east-1.api.aws",
 				},
 				endpointKey{
 					Region: "us-east-1",
@@ -6884,7 +7007,7 @@ var awsPartition = partition{
 					Region:  "us-east-1",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.us-east-1.aws",
+					Hostname: "ec2.us-east-1.api.aws",
 				},
 				endpointKey{
 					Region:  "us-east-1",
@@ -6899,7 +7022,7 @@ var awsPartition = partition{
 					Region:  "us-east-2",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.us-east-2.aws",
+					Hostname: "ec2.us-east-2.api.aws",
 				},
 				endpointKey{
 					Region:  "us-east-2",
@@ -6923,7 +7046,7 @@ var awsPartition = partition{
 					Region:  "us-west-2",
 					Variant: dualStackVariant,
 				}: endpoint{
-					Hostname: "api.ec2.us-west-2.aws",
+					Hostname: "ec2.us-west-2.api.aws",
 				},
 				endpointKey{
 					Region:  "us-west-2",
@@ -7100,6 +7223,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "ap-southeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-3",
+				}: endpoint{},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{},
@@ -8731,6 +8857,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "ap-southeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-3",
+				}: endpoint{},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{},
@@ -10346,6 +10475,14 @@ var awsPartition = partition{
 			},
 		},
 		"health": service{
+			PartitionEndpoint: "aws-global",
+			IsRegionalized:    boxedFalse,
+			Defaults: endpointDefaults{
+				defaultKey{}: endpoint{
+					SSLCommonName: "health.us-east-1.amazonaws.com",
+					Protocols:     []string{"https"},
+				},
+			},
 			Endpoints: serviceEndpoints{
 				endpointKey{
 					Region: "fips-us-east-2",
@@ -10462,6 +10599,9 @@ var awsPartition = partition{
 		},
 		"identity-chime": service{
 			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "eu-central-1",
+				}: endpoint{},
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
@@ -12584,6 +12724,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "ap-southeast-2",
 				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-3",
+				}: endpoint{},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{},
@@ -13629,6 +13772,9 @@ var awsPartition = partition{
 		},
 		"messaging-chime": service{
 			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "eu-central-1",
+				}: endpoint{},
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
@@ -15440,6 +15586,25 @@ var awsPartition = partition{
 				}: endpoint{},
 			},
 		},
+		"proton": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "ap-northeast-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-west-2",
+				}: endpoint{},
+			},
+		},
 		"qldb": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{
@@ -16476,6 +16641,70 @@ var awsPartition = partition{
 				},
 			},
 		},
+		"resiliencehub": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "af-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-northeast-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-northeast-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "ca-central-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-central-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-north-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-3",
+				}: endpoint{},
+				endpointKey{
+					Region: "me-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "sa-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-west-2",
+				}: endpoint{},
+			},
+		},
 		"resource-groups": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{
@@ -20784,6 +21013,42 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "eu-west-3",
 				}: endpoint{},
+				endpointKey{
+					Region: "fips-us-east-1",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-east-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-east-1",
+					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "fips-us-east-2",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-east-2.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-east-2",
+					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "fips-us-west-1",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-west-1",
+					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "fips-us-west-2",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-west-2.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-west-2",
+					},
+					Deprecated: boxedTrue,
+				},
 				endpointKey{
 					Region: "me-south-1",
 				}: endpoint{},
@@ -20793,15 +21058,39 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "us-east-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-east-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-east-1.amazonaws.com",
+				},
 				endpointKey{
 					Region: "us-east-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-east-2",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-east-2.amazonaws.com",
+				},
 				endpointKey{
 					Region: "us-west-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-west-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-west-1.amazonaws.com",
+				},
 				endpointKey{
 					Region: "us-west-2",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-west-2",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-west-2.amazonaws.com",
+				},
 			},
 		},
 		"tagging": service{
@@ -21750,6 +22039,23 @@ var awsPartition = partition{
 						Region: "ap-southeast-2",
 					},
 				},
+				endpointKey{
+					Region: "ap-southeast-3",
+				}: endpoint{
+					Hostname: "waf-regional.ap-southeast-3.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "ap-southeast-3",
+					},
+				},
+				endpointKey{
+					Region:  "ap-southeast-3",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "waf-regional-fips.ap-southeast-3.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "ap-southeast-3",
+					},
+				},
 				endpointKey{
 					Region: "ca-central-1",
 				}: endpoint{
@@ -21941,6 +22247,15 @@ var awsPartition = partition{
 					},
 					Deprecated: boxedTrue,
 				},
+				endpointKey{
+					Region: "fips-ap-southeast-3",
+				}: endpoint{
+					Hostname: "waf-regional-fips.ap-southeast-3.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "ap-southeast-3",
+					},
+					Deprecated: boxedTrue,
+				},
 				endpointKey{
 					Region: "fips-ca-central-1",
 				}: endpoint{
@@ -22162,6 +22477,64 @@ var awsPartition = partition{
 				},
 			},
 		},
+		"wellarchitected": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "ap-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-northeast-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-northeast-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "ap-southeast-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "ca-central-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-central-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-north-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "eu-west-3",
+				}: endpoint{},
+				endpointKey{
+					Region: "me-south-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "sa-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-east-2",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region: "us-west-2",
+				}: endpoint{},
+			},
+		},
 		"wisdom": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{
@@ -22649,9 +23022,21 @@ var awscnPartition = partition{
 				endpointKey{
 					Region: "cn-north-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "cn-north-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.cn-north-1.api.amazonwebservices.com.cn",
+				},
 				endpointKey{
 					Region: "cn-northwest-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "cn-northwest-1",
+					Variant: dualStackVariant,
+				}: endpoint{
+					Hostname: "appmesh.cn-northwest-1.api.amazonwebservices.com.cn",
+				},
 			},
 		},
 		"appsync": service{
@@ -27477,6 +27862,13 @@ var awsusgovPartition = partition{
 				},
 			},
 		},
+		"robomaker": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "us-gov-west-1",
+				}: endpoint{},
+			},
+		},
 		"route53": service{
 			PartitionEndpoint: "aws-us-gov-global",
 			IsRegionalized:    boxedFalse,
@@ -28596,12 +28988,42 @@ var awsusgovPartition = partition{
 		},
 		"synthetics": service{
 			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "fips-us-gov-east-1",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-gov-east-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-gov-east-1",
+					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "fips-us-gov-west-1",
+				}: endpoint{
+					Hostname: "synthetics-fips.us-gov-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-gov-west-1",
+					},
+					Deprecated: boxedTrue,
+				},
 				endpointKey{
 					Region: "us-gov-east-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-gov-east-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-gov-east-1.amazonaws.com",
+				},
 				endpointKey{
 					Region: "us-gov-west-1",
 				}: endpoint{},
+				endpointKey{
+					Region:  "us-gov-west-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "synthetics-fips.us-gov-west-1.amazonaws.com",
+				},
 			},
 		},
 		"tagging": service{
@@ -28966,6 +29388,13 @@ var awsisoPartition = partition{
 				}: endpoint{},
 			},
 		},
+		"appconfigdata": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "us-iso-west-1",
+				}: endpoint{},
+			},
+		},
 		"application-autoscaling": service{
 			Defaults: endpointDefaults{
 				defaultKey{}: endpoint{
@@ -29883,6 +30312,28 @@ var awsisobPartition = partition{
 				}: endpoint{},
 			},
 		},
+		"elasticfilesystem": service{
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "fips-us-isob-east-1",
+				}: endpoint{
+					Hostname: "elasticfilesystem-fips.us-isob-east-1.sc2s.sgov.gov",
+					CredentialScope: credentialScope{
+						Region: "us-isob-east-1",
+					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "us-isob-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region:  "us-isob-east-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "elasticfilesystem-fips.us-isob-east-1.sc2s.sgov.gov",
+				},
+			},
+		},
 		"elasticloadbalancing": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{

vendor/github.com/aws/aws-sdk-go/aws/version.go

@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.44.5"
+const SDKVersion = "1.44.20"

vendor/github.com/aws/aws-sdk-go/service/kms/api.go

@@ -912,12 +912,12 @@ func (c *KMS) CreateKeyRequest(input *CreateKeyInput) (req *request.Request, out
 // and verify. You can't change these properties after the KMS key is created.
 //
 // Asymmetric KMS keys contain an RSA key pair or an Elliptic Curve (ECC) key
-// pair. The private key in an asymmetric KMS key never leaves AWS KMS unencrypted.
+// pair. The private key in an asymmetric KMS key never leaves KMS unencrypted.
 // However, you can use the GetPublicKey operation to download the public key
-// so it can be used outside of AWS KMS. KMS keys with RSA key pairs can be
-// used to encrypt or decrypt data or sign and verify messages (but not both).
-// KMS keys with ECC key pairs can be used only to sign and verify messages.
-// For information about asymmetric KMS keys, see Asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html)
+// so it can be used outside of KMS. KMS keys with RSA key pairs can be used
+// to encrypt or decrypt data or sign and verify messages (but not both). KMS
+// keys with ECC key pairs can be used only to sign and verify messages. For
+// information about asymmetric KMS keys, see Asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html)
 // in the Key Management Service Developer Guide.
 //
 // HMAC KMS key
@@ -1191,8 +1191,8 @@ func (c *KMS) DecryptRequest(input *DecryptInput) (req *request.Request, output
 //
 // The Decrypt operation also decrypts ciphertext that was encrypted outside
 // of KMS by the public key in an KMS asymmetric KMS key. However, it cannot
-// decrypt symmetric ciphertext produced by other libraries, such as the Amazon
-// Web Services Encryption SDK (https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/)
+// decrypt ciphertext produced by other libraries, such as the Amazon Web Services
+// Encryption SDK (https://docs.aws.amazon.com/encryption-sdk/latest/developer-guide/)
 // or Amazon S3 client-side encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingClientSideEncryption.html).
 // These libraries return a ciphertext format that is incompatible with KMS.
 //
@@ -2195,16 +2195,27 @@ func (c *KMS) DisableKeyRotationRequest(input *DisableKeyRotationInput) (req *re
 // DisableKeyRotation API operation for AWS Key Management Service.
 //
 // Disables automatic rotation of the key material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html)
-// for the specified symmetric encryption KMS key.
+// of the specified symmetric encryption KMS key.
 //
-// You cannot enable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
+// Automatic key rotation is supported only on symmetric encryption KMS keys.
+// You cannot enable or disable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
 // HMAC KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/hmac.html),
 // KMS keys with imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html),
 // or KMS keys in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html).
-// To enable or disable automatic rotation of a set of related multi-Region
-// keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
+// The key rotation status of these KMS keys is always false. To enable or disable
+// automatic rotation of a set of related multi-Region keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
 // set the property on the primary key.
 //
+// You can enable (EnableKeyRotation) and disable automatic rotation of the
+// key material in customer managed KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#customer-cmk).
+// Key material rotation of Amazon Web Services managed KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk)
+// is not configurable. KMS always rotates the key material for every year.
+// Rotation of Amazon Web Services owned KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-owned-cmk)
+// varies.
+//
+// In May 2022, KMS changed the rotation schedule for Amazon Web Services managed
+// keys from every three years to every year. For details, see EnableKeyRotation.
+//
 // The KMS key that you use for this operation must be in a compatible key state.
 // For details, see Key states of KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html)
 // in the Key Management Service Developer Guide.
@@ -2589,16 +2600,41 @@ func (c *KMS) EnableKeyRotationRequest(input *EnableKeyRotationInput) (req *requ
 // EnableKeyRotation API operation for AWS Key Management Service.
 //
 // Enables automatic rotation of the key material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html)
-// for the specified symmetric encryption KMS key.
+// of the specified symmetric encryption KMS key.
 //
-// You cannot enable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
+// When you enable automatic rotation of acustomer managed KMS key (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#customer-cmk),
+// KMS rotates the key material of the KMS key one year (approximately 365 days)
+// from the enable date and every year thereafter. You can monitor rotation
+// of the key material for your KMS keys in CloudTrail and Amazon CloudWatch.
+// To disable rotation of the key material in a customer managed KMS key, use
+// the DisableKeyRotation operation.
+//
+// Automatic key rotation is supported only on symmetric encryption KMS keys
+// (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#symmetric-cmks).
+// You cannot enable or disable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
 // HMAC KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/hmac.html),
 // KMS keys with imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html),
 // or KMS keys in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html).
-// To enable or disable automatic rotation of a set of related multi-Region
-// keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
+// The key rotation status of these KMS keys is always false. To enable or disable
+// automatic rotation of a set of related multi-Region keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
 // set the property on the primary key.
 //
+// You cannot enable or disable automatic rotation Amazon Web Services managed
+// KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk).
+// KMS always rotates the key material of Amazon Web Services managed keys every
+// year. Rotation of Amazon Web Services owned KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-owned-cmk)
+// varies.
+//
+// In May 2022, KMS changed the rotation schedule for Amazon Web Services managed
+// keys from every three years (approximately 1,095 days) to every year (approximately
+// 365 days).
+//
+// New Amazon Web Services managed keys are automatically rotated one year after
+// they are created, and approximately every year thereafter.
+//
+// Existing Amazon Web Services managed keys are automatically rotated one year
+// after their most recent rotation, and every year thereafter.
+//
 // The KMS key that you use for this operation must be in a compatible key state.
 // For details, see Key states of KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html)
 // in the Key Management Service Developer Guide.
@@ -3490,14 +3526,16 @@ func (c *KMS) GenerateDataKeyWithoutPlaintextRequest(input *GenerateDataKeyWitho
 //
 // This operation is useful for systems that need to encrypt data at some point,
 // but not immediately. When you need to encrypt the data, you call the Decrypt
-// operation on the encrypted copy of the key. It's also useful in distributed
-// systems with different levels of trust. For example, you might store encrypted
-// data in containers. One component of your system creates new containers and
-// stores an encrypted data key with each container. Then, a different component
-// puts the data into the containers. That component first decrypts the data
-// key, uses the plaintext data key to encrypt data, puts the encrypted data
-// into the container, and then destroys the plaintext data key. In this system,
-// the component that creates the containers never sees the plaintext data key.
+// operation on the encrypted copy of the key.
+//
+// It's also useful in distributed systems with different levels of trust. For
+// example, you might store encrypted data in containers. One component of your
+// system creates new containers and stores an encrypted data key with each
+// container. Then, a different component puts the data into the containers.
+// That component first decrypts the data key, uses the plaintext data key to
+// encrypt data, puts the encrypted data into the container, and then destroys
+// the plaintext data key. In this system, the component that creates the containers
+// never sees the plaintext data key.
 //
 // To request an asymmetric data key pair, use the GenerateDataKeyPair or GenerateDataKeyPairWithoutPlaintext
 // operations.
@@ -3672,6 +3710,13 @@ func (c *KMS) GenerateMacRequest(input *GenerateMacInput) (req *request.Request,
 // KMS support for HMAC KMS keys. For details, see HMAC keys in KMS (https://docs.aws.amazon.com/kms/latest/developerguide/hmac.html)
 // in the Key Management Service Developer Guide .
 //
+// Best practices recommend that you limit the time during which any signing
+// mechanism, including an HMAC, is effective. This deters an attack where the
+// actor uses a signed message to establish validity repeatedly or long after
+// the message is superseded. HMAC tags do not include a timestamp, but you
+// can include a timestamp in the token or message to help you detect when its
+// time to refresh the HMAC.
+//
 // The KMS key that you use for this operation must be in a compatible key state.
 // For details, see Key states of KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html)
 // in the Key Management Service Developer Guide.
@@ -4038,14 +4083,30 @@ func (c *KMS) GetKeyRotationStatusRequest(input *GetKeyRotationStatusInput) (req
 // material (https://docs.aws.amazon.com/kms/latest/developerguide/rotate-keys.html)
 // is enabled for the specified KMS key.
 //
-// You cannot enable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
+// When you enable automatic rotation for customer managed KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#customer-cmk),
+// KMS rotates the key material of the KMS key one year (approximately 365 days)
+// from the enable date and every year thereafter. You can monitor rotation
+// of the key material for your KMS keys in CloudTrail and Amazon CloudWatch.
+//
+// Automatic key rotation is supported only on symmetric encryption KMS keys
+// (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#symmetric-cmks).
+// You cannot enable or disable automatic rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
 // HMAC KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/hmac.html),
 // KMS keys with imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html),
 // or KMS keys in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html).
-// To enable or disable automatic rotation of a set of related multi-Region
-// keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
-// set the property on the primary key. The key rotation status for these KMS
-// keys is always false.
+// The key rotation status of these KMS keys is always false. To enable or disable
+// automatic rotation of a set of related multi-Region keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
+// set the property on the primary key..
+//
+// You can enable (EnableKeyRotation) and disable automatic rotation (DisableKeyRotation)
+// of the key material in customer managed KMS keys. Key material rotation of
+// Amazon Web Services managed KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#aws-managed-cmk)
+// is not configurable. KMS always rotates the key material in Amazon Web Services
+// managed KMS keys every year. The key rotation status for Amazon Web Services
+// managed KMS keys is always true.
+//
+// In May 2022, KMS changed the rotation schedule for Amazon Web Services managed
+// keys from every three years to every year. For details, see EnableKeyRotation.
 //
 // The KMS key that you use for this operation must be in a compatible key state.
 // For details, see Key states of KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/key-state.html)
@@ -4053,11 +4114,15 @@ func (c *KMS) GetKeyRotationStatusRequest(input *GetKeyRotationStatusInput) (req
 //
 //    * Disabled: The key rotation status does not change when you disable a
 //    KMS key. However, while the KMS key is disabled, KMS does not rotate the
-//    key material.
+//    key material. When you re-enable the KMS key, rotation resumes. If the
+//    key material in the re-enabled KMS key hasn't been rotated in one year,
+//    KMS rotates it immediately, and every year thereafter. If it's been less
+//    than a year since the key material in the re-enabled KMS key was rotated,
+//    the KMS key resumes its prior rotation schedule.
 //
 //    * Pending deletion: While a KMS key is pending deletion, its key rotation
 //    status is false and KMS does not rotate the key material. If you cancel
-//    the deletion, the original key rotation status is restored.
+//    the deletion, the original key rotation status returns to true.
 //
 // Cross-account use: Yes. To perform this operation on a KMS key in a different
 // Amazon Web Services account, specify the key ARN in the value of the KeyId
@@ -6644,6 +6709,12 @@ func (c *KMS) SignRequest(input *SignInput) (req *request.Request, output *SignO
 // When signing a message, be sure to record the KMS key and the signing algorithm.
 // This information is required to verify the signature.
 //
+// Best practices recommend that you limit the time during which any signature
+// is effective. This deters an attack where the actor uses a signed message
+// to establish validity repeatedly or long after the message is superseded.
+// Signatures do not include a timestamp, but you can include a timestamp in
+// the signed message to help you detect when its time to refresh the signature.
+//
 // To verify the signature that this operation generates, use the Verify operation.
 // Or use the GetPublicKey operation to download the public key and then use
 // the public key to verify the signature outside of KMS.
@@ -9242,11 +9313,11 @@ type CreateKeyInput struct {
 	// in the Key Management Service Developer Guide .
 	//
 	// The KeySpec determines whether the KMS key contains a symmetric key or an
-	// asymmetric key pair. It also determines the algorithms that the KMS key supports.
-	// You can't change the KeySpec after the KMS key is created. To further restrict
-	// the algorithms that can be used with the KMS key, use a condition key in
-	// its key policy or IAM policy. For more information, see kms:EncryptionAlgorithm
-	// (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-encryption-algorithm),
+	// asymmetric key pair. It also determines the cryptographic algorithms that
+	// the KMS key supports. You can't change the KeySpec after the KMS key is created.
+	// To further restrict the algorithms that can be used with the KMS key, use
+	// a condition key in its key policy or IAM policy. For more information, see
+	// kms:EncryptionAlgorithm (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-encryption-algorithm),
 	// kms:MacAlgorithm (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-mac-algorithm)
 	// or kms:Signing Algorithm (https://docs.aws.amazon.com/kms/latest/developerguide/policy-conditions.html#conditions-kms-signing-algorithm)
 	// in the Key Management Service Developer Guide .
@@ -9307,9 +9378,9 @@ type CreateKeyInput struct {
 	// This value creates a primary key, not a replica. To create a replica key,
 	// use the ReplicateKey operation.
 	//
-	// You can create a symmetric or asymmetric multi-Region key, and you can create
-	// a multi-Region key with imported key material. However, you cannot create
-	// a multi-Region key in a custom key store.
+	// You can create a multi-Region version of a symmetric encryption KMS key,
+	// an HMAC KMS key, an asymmetric KMS key, or a KMS key with imported key material.
+	// However, you cannot create a multi-Region key in a custom key store.
 	MultiRegion *bool `type:"boolean"`
 
 	// The source of the key material for the KMS key. You cannot change the origin
@@ -9329,11 +9400,14 @@ type CreateKeyInput struct {
 	// KMS keys.
 	Origin *string `type:"string" enum:"OriginType"`
 
-	// The key policy to attach to the KMS key.
+	// The key policy to attach to the KMS key. If you do not specify a key policy,
+	// KMS attaches a default key policy to the KMS key. For more information, see
+	// Default key policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default)
+	// in the Key Management Service Developer Guide.
 	//
 	// If you provide a key policy, it must meet the following criteria:
 	//
-	//    * If you don't set BypassPolicyLockoutSafetyCheck to true, the key policy
+	//    * If you don't set BypassPolicyLockoutSafetyCheck to True, the key policy
 	//    must allow the principal that is making the CreateKey request to make
 	//    a subsequent PutKeyPolicy request on the KMS key. This reduces the risk
 	//    that the KMS key becomes unmanageable. For more information, refer to
@@ -9349,11 +9423,18 @@ type CreateKeyInput struct {
 	//    visible (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency)
 	//    in the Amazon Web Services Identity and Access Management User Guide.
 	//
-	// If you do not provide a key policy, KMS attaches a default key policy to
-	// the KMS key. For more information, see Default Key Policy (https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html#key-policy-default)
-	// in the Key Management Service Developer Guide.
+	// A key policy document must conform to the following rules.
 	//
-	// The key policy size quota is 32 kilobytes (32768 bytes).
+	//    * Up to 32 kilobytes (32768 bytes)
+	//
+	//    * Must be UTF-8 encoded
+	//
+	//    * The only Unicode characters that are permitted in a key policy document
+	//    are the horizontal tab (U+0009), linefeed (U+000A), carriage return (U+000D),
+	//    and characters in the range U+0020 to U+00FF.
+	//
+	//    * The Sid element in a key policy statement can include spaces. (Spaces
+	//    are prohibited in the Sid element of an IAM policy document.)
 	//
 	// For help writing and formatting a JSON policy document, see the IAM JSON
 	// Policy Reference (https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies.html)
@@ -11098,13 +11179,13 @@ func (s EnableKeyOutput) GoString() string {
 type EnableKeyRotationInput struct {
 	_ struct{} `type:"structure"`
 
-	// Identifies a symmetric encryption KMS key. You cannot enable automatic rotation
-	// of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
+	// Identifies a symmetric encryption KMS key. You cannot enable or disable automatic
+	// rotation of asymmetric KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/symmetric-asymmetric.html),
 	// HMAC KMS keys (https://docs.aws.amazon.com/kms/latest/developerguide/hmac.html),
 	// KMS keys with imported key material (https://docs.aws.amazon.com/kms/latest/developerguide/importing-keys.html),
 	// or KMS keys in a custom key store (https://docs.aws.amazon.com/kms/latest/developerguide/custom-key-store-overview.html).
-	// To enable or disable automatic rotation of a set of related multi-Region
-	// keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
+	// The key rotation status of these KMS keys is always false. To enable or disable
+	// automatic rotation of a set of related multi-Region keys (https://docs.aws.amazon.com/kms/latest/developerguide/multi-region-keys-manage.html#multi-region-rotate),
 	// set the property on the primary key.
 	//
 	// Specify the key ID or key ARN of the KMS key.
@@ -15929,9 +16010,18 @@ type PutKeyPolicyInput struct {
 	//    visible (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency)
 	//    in the Amazon Web Services Identity and Access Management User Guide.
 	//
-	// The key policy cannot exceed 32 kilobytes (32768 bytes). For more information,
-	// see Resource Quotas (https://docs.aws.amazon.com/kms/latest/developerguide/resource-limits.html)
-	// in the Key Management Service Developer Guide.
+	// A key policy document must conform to the following rules.
+	//
+	//    * Up to 32 kilobytes (32768 bytes)
+	//
+	//    * Must be UTF-8 encoded
+	//
+	//    * The only Unicode characters that are permitted in a key policy document
+	//    are the horizontal tab (U+0009), linefeed (U+000A), carriage return (U+000D),
+	//    and characters in the range U+0020 to U+00FF.
+	//
+	//    * The Sid element in a key policy statement can include spaces. (Spaces
+	//    are prohibited in the Sid element of an IAM policy document.)
 	//
 	// Policy is a required field
 	Policy *string `min:"1" type:"string" required:"true"`
@@ -16391,7 +16481,18 @@ type ReplicateKeyInput struct {
 	//    visible (https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_general.html#troubleshoot_general_eventual-consistency)
 	//    in the Identity and Access Management User Guide .
 	//
-	//    * The key policy size quota is 32 kilobytes (32768 bytes).
+	// A key policy document must conform to the following rules.
+	//
+	//    * Up to 32 kilobytes (32768 bytes)
+	//
+	//    * Must be UTF-8 encoded
+	//
+	//    * The only Unicode characters that are permitted in a key policy document
+	//    are the horizontal tab (U+0009), linefeed (U+000A), carriage return (U+000D),
+	//    and characters in the range U+0020 to U+00FF.
+	//
+	//    * The Sid element in a key policy statement can include spaces. (Spaces
+	//    are prohibited in the Sid element of an IAM policy document.)
 	Policy *string `min:"1" type:"string"`
 
 	// The Region ID of the Amazon Web Services Region for this replica key.
@@ -16818,9 +16919,9 @@ type ScheduleKeyDeletionInput struct {
 	// The waiting period, specified in number of days. After the waiting period
 	// ends, KMS deletes the KMS key.
 	//
-	// If the KMS key is a multi-Region primary key with replicas, the waiting period
-	// begins when the last of its replica keys is deleted. Otherwise, the waiting
-	// period begins immediately.
+	// If the KMS key is a multi-Region primary key with replica keys, the waiting
+	// period begins when the last of its replica keys is deleted. Otherwise, the
+	// waiting period begins immediately.
 	//
 	// This value is optional. If you include a value, it must be between 7 and
 	// 30, inclusive. If you do not include a value, it defaults to 30.

vendor/github.com/aws/aws-sdk-go/service/kms/doc.go

@@ -30,11 +30,11 @@
 // see Service endpoints (https://docs.aws.amazon.com/general/latest/gr/kms.html#kms_region)
 // in the Key Management Service topic of the Amazon Web Services General Reference.
 //
-// Clients must support TLS (Transport Layer Security) 1.0. We recommend TLS
-// 1.2. Clients must also support cipher suites with Perfect Forward Secrecy
-// (PFS) such as Ephemeral Diffie-Hellman (DHE) or Elliptic Curve Ephemeral
-// Diffie-Hellman (ECDHE). Most modern systems such as Java 7 and later support
-// these modes.
+// All KMS API calls must be signed and be transmitted using Transport Layer
+// Security (TLS). KMS recommends you always use the latest supported TLS version.
+// Clients must also support cipher suites with Perfect Forward Secrecy (PFS)
+// such as Ephemeral Diffie-Hellman (DHE) or Elliptic Curve Ephemeral Diffie-Hellman
+// (ECDHE). Most modern systems such as Java 7 and later support these modes.
 //
 // Signing Requests
 //

vendor/github.com/aws/aws-sdk-go/service/sts/api.go

@@ -1279,6 +1279,12 @@ func (c *STS) GetSessionTokenRequest(input *GetSessionTokenInput) (req *request.
 // and Comparing the Amazon Web Services STS API operations (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_request.html#stsapi_comparison)
 // in the IAM User Guide.
 //
+// No permissions are required for users to perform this operation. The purpose
+// of the sts:GetSessionToken operation is to authenticate the user using MFA.
+// You cannot use policies to control authentication operations. For more information,
+// see Permissions for GetSessionToken (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_control-access_getsessiontoken.html)
+// in the IAM User Guide.
+//
 // Session Duration
 //
 // The GetSessionToken operation must be called by using the long-term Amazon

vendor/github.com/bits-and-blooms/bitset/.gitignore

@@ -1,26 +0,0 @@
-# Compiled Object files, Static and Dynamic libs (Shared Objects)
-*.o
-*.a
-*.so
-
-# Folders
-_obj
-_test
-
-# Architecture specific extensions/prefixes
-*.[568vq]
-[568vq].out
-
-*.cgo1.go
-*.cgo2.c
-_cgo_defun.c
-_cgo_gotypes.go
-_cgo_export.*
-
-_testmain.go
-
-*.exe
-*.test
-*.prof
-
-target

vendor/github.com/bits-and-blooms/bitset/.travis.yml

@@ -1,37 +0,0 @@
-language: go
-
-sudo: false
-
-branches:
-  except:
-    - release
-
-branches:
-  only:
-    - master
-    - travis
-
-go:
-  - "1.11.x"
-  - tip
-
-matrix:
-  allow_failures:
-    - go: tip
-
-before_install:
-  - if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi;
-  - if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi;
-  - go get github.com/mattn/goveralls
-
-before_script:
-  - make deps
-
-script:
-  - make qa
-
-after_failure:
-  - cat ./target/test/report.xml
-
-after_success:
-  - if [ "$TRAVIS_GO_VERSION" = "1.11.1" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi;

vendor/github.com/bits-and-blooms/bitset/README.md

@@ -1,93 +0,0 @@
-# bitset
-
-*Go language library to map between non-negative integers and boolean values*
-
-[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest)
-[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset)
-[![PkgGoDev](https://pkg.go.dev/badge/github.com/bits-and-blooms/bitset?tab=doc)](https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc)
-
-
-## Description
-
-Package bitset implements bitsets, a mapping between non-negative integers and boolean values.
-It should be more efficient than map[uint] bool.
-
-It provides methods for setting, clearing, flipping, and testing individual integers.
-
-But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits.
-
-BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used.
-
-Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining.
-
-### Example use:
-
-```go
-package main
-
-import (
-	"fmt"
-	"math/rand"
-
-	"github.com/bits-and-blooms/bitset"
-)
-
-func main() {
-	fmt.Printf("Hello from BitSet!\n")
-	var b bitset.BitSet
-	// play some Go Fish
-	for i := 0; i < 100; i++ {
-		card1 := uint(rand.Intn(52))
-		card2 := uint(rand.Intn(52))
-		b.Set(card1)
-		if b.Test(card2) {
-			fmt.Println("Go Fish!")
-		}
-		b.Clear(card1)
-	}
-
-	// Chaining
-	b.Set(10).Set(11)
-
-	for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) {
-		fmt.Println("The following bit is set:", i)
-	}
-	if b.Intersection(bitset.New(100).Set(10)).Count() == 1 {
-		fmt.Println("Intersection works.")
-	} else {
-		fmt.Println("Intersection doesn't work???")
-	}
-}
-```
-
-As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets.
-
-Package documentation is at: https://pkg.go.dev/github.com/bits-and-blooms/bitset?tab=doc
-
-## Memory Usage
-
-The memory usage of a bitset using N bits is at least N/8 bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring).
-
-## Implementation Note
-
-Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed.
-
-It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped.
-
-## Installation
-
-```bash
-go get github.com/bits-and-blooms/bitset
-```
-
-## Contributing
-
-If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)")
-
-## Running all tests
-
-Before committing the code, please check if it passes tests, has adequate coverage, etc.
-```bash
-go test
-go test -cover
-```

vendor/github.com/bits-and-blooms/bitset/azure-pipelines.yml

@@ -1,39 +0,0 @@
-# Go
-# Build your Go project.
-# Add steps that test, save build artifacts, deploy, and more:
-# https://docs.microsoft.com/azure/devops/pipelines/languages/go
-
-trigger:
-- master
-
-pool:
-  vmImage: 'Ubuntu-16.04'
-
-variables:
-  GOBIN:  '$(GOPATH)/bin' # Go binaries path
-  GOROOT: '/usr/local/go1.11' # Go installation path
-  GOPATH: '$(system.defaultWorkingDirectory)/gopath' # Go workspace path
-  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' # Path to the module's code
-
-steps:
-- script: |
-    mkdir -p '$(GOBIN)'
-    mkdir -p '$(GOPATH)/pkg'
-    mkdir -p '$(modulePath)'
-    shopt -s extglob
-    shopt -s dotglob
-    mv !(gopath) '$(modulePath)'
-    echo '##vso[task.prependpath]$(GOBIN)'
-    echo '##vso[task.prependpath]$(GOROOT)/bin'
-  displayName: 'Set up the Go workspace'
-
-- script: |
-    go version
-    go get -v -t -d ./...
-    if [ -f Gopkg.toml ]; then
-        curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-        dep ensure
-    fi
-    go build -v .
-  workingDirectory: '$(modulePath)'
-  displayName: 'Get dependencies, then build'

vendor/github.com/bits-and-blooms/bitset/bitset.go

@@ -1,952 +0,0 @@
-/*
-Package bitset implements bitsets, a mapping
-between non-negative integers and boolean values. It should be more
-efficient than map[uint] bool.
-
-It provides methods for setting, clearing, flipping, and testing
-individual integers.
-
-But it also provides set intersection, union, difference,
-complement, and symmetric operations, as well as tests to
-check whether any, all, or no bits are set, and querying a
-bitset's current length and number of positive bits.
-
-BitSets are expanded to the size of the largest set bit; the
-memory allocation is approximately Max bits, where Max is
-the largest set bit. BitSets are never shrunk. On creation,
-a hint can be given for the number of bits that will be used.
-
-Many of the methods, including Set,Clear, and Flip, return
-a BitSet pointer, which allows for chaining.
-
-Example use:
-
-	import "bitset"
-	var b BitSet
-	b.Set(10).Set(11)
-	if b.Test(1000) {
-		b.Clear(1000)
-	}
-	if B.Intersection(bitset.New(100).Set(10)).Count() > 1 {
-		fmt.Println("Intersection works.")
-	}
-
-As an alternative to BitSets, one should check out the 'big' package,
-which provides a (less set-theoretical) view of bitsets.
-
-*/
-package bitset
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/base64"
-	"encoding/binary"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"strconv"
-)
-
-// the wordSize of a bit set
-const wordSize = uint(64)
-
-// log2WordSize is lg(wordSize)
-const log2WordSize = uint(6)
-
-// allBits has every bit set
-const allBits uint64 = 0xffffffffffffffff
-
-// default binary BigEndian
-var binaryOrder binary.ByteOrder = binary.BigEndian
-
-// default json encoding base64.URLEncoding
-var base64Encoding = base64.URLEncoding
-
-// Base64StdEncoding Marshal/Unmarshal BitSet with base64.StdEncoding(Default: base64.URLEncoding)
-func Base64StdEncoding() { base64Encoding = base64.StdEncoding }
-
-// LittleEndian Marshal/Unmarshal Binary as Little Endian(Default: binary.BigEndian)
-func LittleEndian() { binaryOrder = binary.LittleEndian }
-
-// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0.
-type BitSet struct {
-	length uint
-	set    []uint64
-}
-
-// Error is used to distinguish errors (panics) generated in this package.
-type Error string
-
-// safeSet will fixup b.set to be non-nil and return the field value
-func (b *BitSet) safeSet() []uint64 {
-	if b.set == nil {
-		b.set = make([]uint64, wordsNeeded(0))
-	}
-	return b.set
-}
-
-// From is a constructor used to create a BitSet from an array of integers
-func From(buf []uint64) *BitSet {
-	return &BitSet{uint(len(buf)) * 64, buf}
-}
-
-// Bytes returns the bitset as array of integers
-func (b *BitSet) Bytes() []uint64 {
-	return b.set
-}
-
-// wordsNeeded calculates the number of words needed for i bits
-func wordsNeeded(i uint) int {
-	if i > (Cap() - wordSize + 1) {
-		return int(Cap() >> log2WordSize)
-	}
-	return int((i + (wordSize - 1)) >> log2WordSize)
-}
-
-// New creates a new BitSet with a hint that length bits will be required
-func New(length uint) (bset *BitSet) {
-	defer func() {
-		if r := recover(); r != nil {
-			bset = &BitSet{
-				0,
-				make([]uint64, 0),
-			}
-		}
-	}()
-
-	bset = &BitSet{
-		length,
-		make([]uint64, wordsNeeded(length)),
-	}
-
-	return bset
-}
-
-// Cap returns the total possible capacity, or number of bits
-func Cap() uint {
-	return ^uint(0)
-}
-
-// Len returns the number of bits in the BitSet.
-// Note the difference to method Count, see example.
-func (b *BitSet) Len() uint {
-	return b.length
-}
-
-// extendSetMaybe adds additional words to incorporate new bits if needed
-func (b *BitSet) extendSetMaybe(i uint) {
-	if i >= b.length { // if we need more bits, make 'em
-		if i >= Cap() {
-			panic("You are exceeding the capacity")
-		}
-		nsize := wordsNeeded(i + 1)
-		if b.set == nil {
-			b.set = make([]uint64, nsize)
-		} else if cap(b.set) >= nsize {
-			b.set = b.set[:nsize] // fast resize
-		} else if len(b.set) < nsize {
-			newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x
-			copy(newset, b.set)
-			b.set = newset
-		}
-		b.length = i + 1
-	}
-}
-
-// Test whether bit i is set.
-func (b *BitSet) Test(i uint) bool {
-	if i >= b.length {
-		return false
-	}
-	return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0
-}
-
-// Set bit i to 1, the capacity of the bitset is automatically
-// increased accordingly.
-// If i>= Cap(), this function will panic.
-// Warning: using a very large value for 'i'
-// may lead to a memory shortage and a panic: the caller is responsible
-// for providing sensible parameters in line with their memory capacity.
-func (b *BitSet) Set(i uint) *BitSet {
-	b.extendSetMaybe(i)
-	b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1))
-	return b
-}
-
-// Clear bit i to 0
-func (b *BitSet) Clear(i uint) *BitSet {
-	if i >= b.length {
-		return b
-	}
-	b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1))
-	return b
-}
-
-// SetTo sets bit i to value.
-// If i>= Cap(), this function will panic.
-// Warning: using a very large value for 'i'
-// may lead to a memory shortage and a panic: the caller is responsible
-// for providing sensible parameters in line with their memory capacity.
-func (b *BitSet) SetTo(i uint, value bool) *BitSet {
-	if value {
-		return b.Set(i)
-	}
-	return b.Clear(i)
-}
-
-// Flip bit at i.
-// If i>= Cap(), this function will panic.
-// Warning: using a very large value for 'i'
-// may lead to a memory shortage and a panic: the caller is responsible
-// for providing sensible parameters in line with their memory capacity.
-func (b *BitSet) Flip(i uint) *BitSet {
-	if i >= b.length {
-		return b.Set(i)
-	}
-	b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1))
-	return b
-}
-
-// FlipRange bit in [start, end).
-// If end>= Cap(), this function will panic.
-// Warning: using a very large value for 'end'
-// may lead to a memory shortage and a panic: the caller is responsible
-// for providing sensible parameters in line with their memory capacity.
-func (b *BitSet) FlipRange(start, end uint) *BitSet {
-	if start >= end {
-		return b
-	}
-
-	b.extendSetMaybe(end - 1)
-	var startWord uint = start >> log2WordSize
-	var endWord uint = end >> log2WordSize
-	b.set[startWord] ^= ^(^uint64(0) << (start & (wordSize - 1)))
-	for i := startWord; i < endWord; i++ {
-		b.set[i] = ^b.set[i]
-	}
-	b.set[endWord] ^= ^uint64(0) >> (-end & (wordSize - 1))
-	return b
-}
-
-// Shrink shrinks BitSet so that the provided value is the last possible
-// set value. It clears all bits > the provided index and reduces the size
-// and length of the set.
-//
-// Note that the parameter value is not the new length in bits: it is the
-// maximal value that can be stored in the bitset after the function call.
-// The new length in bits is the parameter value + 1. Thus it is not possible
-// to use this function to set the length to 0, the minimal value of the length
-// after this function call is 1.
-//
-// A new slice is allocated to store the new bits, so you may see an increase in
-// memory usage until the GC runs. Normally this should not be a problem, but if you
-// have an extremely large BitSet its important to understand that the old BitSet will
-// remain in memory until the GC frees it.
-func (b *BitSet) Shrink(lastbitindex uint) *BitSet {
-	length := lastbitindex + 1
-	idx := wordsNeeded(length)
-	if idx > len(b.set) {
-		return b
-	}
-	shrunk := make([]uint64, idx)
-	copy(shrunk, b.set[:idx])
-	b.set = shrunk
-	b.length = length
-	b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1))))
-	return b
-}
-
-// Compact shrinks BitSet to so that we preserve all set bits, while minimizing
-// memory usage. Compact calls Shrink.
-func (b *BitSet) Compact() *BitSet {
-	idx := len(b.set) - 1
-	for ; idx >= 0 && b.set[idx] == 0; idx-- {
-	}
-	newlength := uint((idx + 1) << log2WordSize)
-	if newlength >= b.length {
-		return b // nothing to do
-	}
-	if newlength > 0 {
-		return b.Shrink(newlength - 1)
-	}
-	// We preserve one word
-	return b.Shrink(63)
-}
-
-// InsertAt takes an index which indicates where a bit should be
-// inserted. Then it shifts all the bits in the set to the left by 1, starting
-// from the given index position, and sets the index position to 0.
-//
-// Depending on the size of your BitSet, and where you are inserting the new entry,
-// this method could be extremely slow and in some cases might cause the entire BitSet
-// to be recopied.
-func (b *BitSet) InsertAt(idx uint) *BitSet {
-	insertAtElement := (idx >> log2WordSize)
-
-	// if length of set is a multiple of wordSize we need to allocate more space first
-	if b.isLenExactMultiple() {
-		b.set = append(b.set, uint64(0))
-	}
-
-	var i uint
-	for i = uint(len(b.set) - 1); i > insertAtElement; i-- {
-		// all elements above the position where we want to insert can simply by shifted
-		b.set[i] <<= 1
-
-		// we take the most significant bit of the previous element and set it as
-		// the least significant bit of the current element
-		b.set[i] |= (b.set[i-1] & 0x8000000000000000) >> 63
-	}
-
-	// generate a mask to extract the data that we need to shift left
-	// within the element where we insert a bit
-	dataMask := ^(uint64(1)<<uint64(idx&(wordSize-1)) - 1)
-
-	// extract that data that we'll shift
-	data := b.set[i] & dataMask
-
-	// set the positions of the data mask to 0 in the element where we insert
-	b.set[i] &= ^dataMask
-
-	// shift data mask to the left and insert its data to the slice element
-	b.set[i] |= data << 1
-
-	// add 1 to length of BitSet
-	b.length++
-
-	return b
-}
-
-// String creates a string representation of the Bitmap
-func (b *BitSet) String() string {
-	// follows code from https://github.com/RoaringBitmap/roaring
-	var buffer bytes.Buffer
-	start := []byte("{")
-	buffer.Write(start)
-	counter := 0
-	i, e := b.NextSet(0)
-	for e {
-		counter = counter + 1
-		// to avoid exhausting the memory
-		if counter > 0x40000 {
-			buffer.WriteString("...")
-			break
-		}
-		buffer.WriteString(strconv.FormatInt(int64(i), 10))
-		i, e = b.NextSet(i + 1)
-		if e {
-			buffer.WriteString(",")
-		}
-	}
-	buffer.WriteString("}")
-	return buffer.String()
-}
-
-// DeleteAt deletes the bit at the given index position from
-// within the bitset
-// All the bits residing on the left of the deleted bit get
-// shifted right by 1
-// The running time of this operation may potentially be
-// relatively slow, O(length)
-func (b *BitSet) DeleteAt(i uint) *BitSet {
-	// the index of the slice element where we'll delete a bit
-	deleteAtElement := i >> log2WordSize
-
-	// generate a mask for the data that needs to be shifted right
-	// within that slice element that gets modified
-	dataMask := ^((uint64(1) << (i & (wordSize - 1))) - 1)
-
-	// extract the data that we'll shift right from the slice element
-	data := b.set[deleteAtElement] & dataMask
-
-	// set the masked area to 0 while leaving the rest as it is
-	b.set[deleteAtElement] &= ^dataMask
-
-	// shift the previously extracted data to the right and then
-	// set it in the previously masked area
-	b.set[deleteAtElement] |= (data >> 1) & dataMask
-
-	// loop over all the consecutive slice elements to copy each
-	// lowest bit into the highest position of the previous element,
-	// then shift the entire content to the right by 1
-	for i := int(deleteAtElement) + 1; i < len(b.set); i++ {
-		b.set[i-1] |= (b.set[i] & 1) << 63
-		b.set[i] >>= 1
-	}
-
-	b.length = b.length - 1
-
-	return b
-}
-
-// NextSet returns the next bit set from the specified index,
-// including possibly the current index
-// along with an error code (true = valid, false = no set bit found)
-// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...}
-//
-// Users concerned with performance may want to use NextSetMany to
-// retrieve several values at once.
-func (b *BitSet) NextSet(i uint) (uint, bool) {
-	x := int(i >> log2WordSize)
-	if x >= len(b.set) {
-		return 0, false
-	}
-	w := b.set[x]
-	w = w >> (i & (wordSize - 1))
-	if w != 0 {
-		return i + trailingZeroes64(w), true
-	}
-	x = x + 1
-	for x < len(b.set) {
-		if b.set[x] != 0 {
-			return uint(x)*wordSize + trailingZeroes64(b.set[x]), true
-		}
-		x = x + 1
-
-	}
-	return 0, false
-}
-
-// NextSetMany returns many next bit sets from the specified index,
-// including possibly the current index and up to cap(buffer).
-// If the returned slice has len zero, then no more set bits were found
-//
-//    buffer := make([]uint, 256) // this should be reused
-//    j := uint(0)
-//    j, buffer = bitmap.NextSetMany(j, buffer)
-//    for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j,buffer) {
-//     for k := range buffer {
-//      do something with buffer[k]
-//     }
-//     j += 1
-//    }
-//
-//
-// It is possible to retrieve all set bits as follow:
-//
-//    indices := make([]uint, bitmap.Count())
-//    bitmap.NextSetMany(0, indices)
-//
-// However if bitmap.Count() is large, it might be preferable to
-// use several calls to NextSetMany, for performance reasons.
-func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) {
-	myanswer := buffer
-	capacity := cap(buffer)
-	x := int(i >> log2WordSize)
-	if x >= len(b.set) || capacity == 0 {
-		return 0, myanswer[:0]
-	}
-	skip := i & (wordSize - 1)
-	word := b.set[x] >> skip
-	myanswer = myanswer[:capacity]
-	size := int(0)
-	for word != 0 {
-		r := trailingZeroes64(word)
-		t := word & ((^word) + 1)
-		myanswer[size] = r + i
-		size++
-		if size == capacity {
-			goto End
-		}
-		word = word ^ t
-	}
-	x++
-	for idx, word := range b.set[x:] {
-		for word != 0 {
-			r := trailingZeroes64(word)
-			t := word & ((^word) + 1)
-			myanswer[size] = r + (uint(x+idx) << 6)
-			size++
-			if size == capacity {
-				goto End
-			}
-			word = word ^ t
-		}
-	}
-End:
-	if size > 0 {
-		return myanswer[size-1], myanswer[:size]
-	}
-	return 0, myanswer[:0]
-}
-
-// NextClear returns the next clear bit from the specified index,
-// including possibly the current index
-// along with an error code (true = valid, false = no bit found i.e. all bits are set)
-func (b *BitSet) NextClear(i uint) (uint, bool) {
-	x := int(i >> log2WordSize)
-	if x >= len(b.set) {
-		return 0, false
-	}
-	w := b.set[x]
-	w = w >> (i & (wordSize - 1))
-	wA := allBits >> (i & (wordSize - 1))
-	index := i + trailingZeroes64(^w)
-	if w != wA && index < b.length {
-		return index, true
-	}
-	x++
-	for x < len(b.set) {
-		index = uint(x)*wordSize + trailingZeroes64(^b.set[x])
-		if b.set[x] != allBits && index < b.length {
-			return index, true
-		}
-		x++
-	}
-	return 0, false
-}
-
-// ClearAll clears the entire BitSet
-func (b *BitSet) ClearAll() *BitSet {
-	if b != nil && b.set != nil {
-		for i := range b.set {
-			b.set[i] = 0
-		}
-	}
-	return b
-}
-
-// wordCount returns the number of words used in a bit set
-func (b *BitSet) wordCount() int {
-	return len(b.set)
-}
-
-// Clone this BitSet
-func (b *BitSet) Clone() *BitSet {
-	c := New(b.length)
-	if b.set != nil { // Clone should not modify current object
-		copy(c.set, b.set)
-	}
-	return c
-}
-
-// Copy into a destination BitSet
-// Returning the size of the destination BitSet
-// like array copy
-func (b *BitSet) Copy(c *BitSet) (count uint) {
-	if c == nil {
-		return
-	}
-	if b.set != nil { // Copy should not modify current object
-		copy(c.set, b.set)
-	}
-	count = c.length
-	if b.length < c.length {
-		count = b.length
-	}
-	return
-}
-
-// Count (number of set bits).
-// Also known as "popcount" or "population count".
-func (b *BitSet) Count() uint {
-	if b != nil && b.set != nil {
-		return uint(popcntSlice(b.set))
-	}
-	return 0
-}
-
-// Equal tests the equivalence of two BitSets.
-// False if they are of different sizes, otherwise true
-// only if all the same bits are set
-func (b *BitSet) Equal(c *BitSet) bool {
-	if c == nil || b == nil {
-		return c == b
-	}
-	if b.length != c.length {
-		return false
-	}
-	if b.length == 0 { // if they have both length == 0, then could have nil set
-		return true
-	}
-	// testing for equality shoud not transform the bitset (no call to safeSet)
-
-	for p, v := range b.set {
-		if c.set[p] != v {
-			return false
-		}
-	}
-	return true
-}
-
-func panicIfNull(b *BitSet) {
-	if b == nil {
-		panic(Error("BitSet must not be null"))
-	}
-}
-
-// Difference of base set and other set
-// This is the BitSet equivalent of &^ (and not)
-func (b *BitSet) Difference(compare *BitSet) (result *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	result = b.Clone() // clone b (in case b is bigger than compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	for i := 0; i < l; i++ {
-		result.set[i] = b.set[i] &^ compare.set[i]
-	}
-	return
-}
-
-// DifferenceCardinality computes the cardinality of the differnce
-func (b *BitSet) DifferenceCardinality(compare *BitSet) uint {
-	panicIfNull(b)
-	panicIfNull(compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	cnt := uint64(0)
-	cnt += popcntMaskSlice(b.set[:l], compare.set[:l])
-	cnt += popcntSlice(b.set[l:])
-	return uint(cnt)
-}
-
-// InPlaceDifference computes the difference of base set and other set
-// This is the BitSet equivalent of &^ (and not)
-func (b *BitSet) InPlaceDifference(compare *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	for i := 0; i < l; i++ {
-		b.set[i] &^= compare.set[i]
-	}
-}
-
-// Convenience function: return two bitsets ordered by
-// increasing length. Note: neither can be nil
-func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) {
-	if a.length <= b.length {
-		ap, bp = a, b
-	} else {
-		ap, bp = b, a
-	}
-	return
-}
-
-// Intersection of base set and other set
-// This is the BitSet equivalent of & (and)
-func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	result = New(b.length)
-	for i, word := range b.set {
-		result.set[i] = word & compare.set[i]
-	}
-	return
-}
-
-// IntersectionCardinality computes the cardinality of the union
-func (b *BitSet) IntersectionCardinality(compare *BitSet) uint {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	cnt := popcntAndSlice(b.set, compare.set)
-	return uint(cnt)
-}
-
-// InPlaceIntersection destructively computes the intersection of
-// base set and the compare set.
-// This is the BitSet equivalent of & (and)
-func (b *BitSet) InPlaceIntersection(compare *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	for i := 0; i < l; i++ {
-		b.set[i] &= compare.set[i]
-	}
-	for i := l; i < len(b.set); i++ {
-		b.set[i] = 0
-	}
-	if compare.length > 0 {
-		b.extendSetMaybe(compare.length - 1)
-	}
-}
-
-// Union of base set and other set
-// This is the BitSet equivalent of | (or)
-func (b *BitSet) Union(compare *BitSet) (result *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	result = compare.Clone()
-	for i, word := range b.set {
-		result.set[i] = word | compare.set[i]
-	}
-	return
-}
-
-// UnionCardinality computes the cardinality of the uniton of the base set
-// and the compare set.
-func (b *BitSet) UnionCardinality(compare *BitSet) uint {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	cnt := popcntOrSlice(b.set, compare.set)
-	if len(compare.set) > len(b.set) {
-		cnt += popcntSlice(compare.set[len(b.set):])
-	}
-	return uint(cnt)
-}
-
-// InPlaceUnion creates the destructive union of base set and compare set.
-// This is the BitSet equivalent of | (or).
-func (b *BitSet) InPlaceUnion(compare *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	if compare.length > 0 {
-		b.extendSetMaybe(compare.length - 1)
-	}
-	for i := 0; i < l; i++ {
-		b.set[i] |= compare.set[i]
-	}
-	if len(compare.set) > l {
-		for i := l; i < len(compare.set); i++ {
-			b.set[i] = compare.set[i]
-		}
-	}
-}
-
-// SymmetricDifference of base set and other set
-// This is the BitSet equivalent of ^ (xor)
-func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	// compare is bigger, so clone it
-	result = compare.Clone()
-	for i, word := range b.set {
-		result.set[i] = word ^ compare.set[i]
-	}
-	return
-}
-
-// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference
-func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint {
-	panicIfNull(b)
-	panicIfNull(compare)
-	b, compare = sortByLength(b, compare)
-	cnt := popcntXorSlice(b.set, compare.set)
-	if len(compare.set) > len(b.set) {
-		cnt += popcntSlice(compare.set[len(b.set):])
-	}
-	return uint(cnt)
-}
-
-// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set
-// This is the BitSet equivalent of ^ (xor)
-func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) {
-	panicIfNull(b)
-	panicIfNull(compare)
-	l := int(compare.wordCount())
-	if l > int(b.wordCount()) {
-		l = int(b.wordCount())
-	}
-	if compare.length > 0 {
-		b.extendSetMaybe(compare.length - 1)
-	}
-	for i := 0; i < l; i++ {
-		b.set[i] ^= compare.set[i]
-	}
-	if len(compare.set) > l {
-		for i := l; i < len(compare.set); i++ {
-			b.set[i] = compare.set[i]
-		}
-	}
-}
-
-// Is the length an exact multiple of word sizes?
-func (b *BitSet) isLenExactMultiple() bool {
-	return b.length%wordSize == 0
-}
-
-// Clean last word by setting unused bits to 0
-func (b *BitSet) cleanLastWord() {
-	if !b.isLenExactMultiple() {
-		b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize)
-	}
-}
-
-// Complement computes the (local) complement of a biset (up to length bits)
-func (b *BitSet) Complement() (result *BitSet) {
-	panicIfNull(b)
-	result = New(b.length)
-	for i, word := range b.set {
-		result.set[i] = ^word
-	}
-	result.cleanLastWord()
-	return
-}
-
-// All returns true if all bits are set, false otherwise. Returns true for
-// empty sets.
-func (b *BitSet) All() bool {
-	panicIfNull(b)
-	return b.Count() == b.length
-}
-
-// None returns true if no bit is set, false otherwise. Returns true for
-// empty sets.
-func (b *BitSet) None() bool {
-	panicIfNull(b)
-	if b != nil && b.set != nil {
-		for _, word := range b.set {
-			if word > 0 {
-				return false
-			}
-		}
-		return true
-	}
-	return true
-}
-
-// Any returns true if any bit is set, false otherwise
-func (b *BitSet) Any() bool {
-	panicIfNull(b)
-	return !b.None()
-}
-
-// IsSuperSet returns true if this is a superset of the other set
-func (b *BitSet) IsSuperSet(other *BitSet) bool {
-	for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) {
-		if !b.Test(i) {
-			return false
-		}
-	}
-	return true
-}
-
-// IsStrictSuperSet returns true if this is a strict superset of the other set
-func (b *BitSet) IsStrictSuperSet(other *BitSet) bool {
-	return b.Count() > other.Count() && b.IsSuperSet(other)
-}
-
-// DumpAsBits dumps a bit set as a string of bits
-func (b *BitSet) DumpAsBits() string {
-	if b.set == nil {
-		return "."
-	}
-	buffer := bytes.NewBufferString("")
-	i := len(b.set) - 1
-	for ; i >= 0; i-- {
-		fmt.Fprintf(buffer, "%064b.", b.set[i])
-	}
-	return buffer.String()
-}
-
-// BinaryStorageSize returns the binary storage requirements
-func (b *BitSet) BinaryStorageSize() int {
-	return binary.Size(uint64(0)) + binary.Size(b.set)
-}
-
-// WriteTo writes a BitSet to a stream
-func (b *BitSet) WriteTo(stream io.Writer) (int64, error) {
-	length := uint64(b.length)
-
-	// Write length
-	err := binary.Write(stream, binaryOrder, length)
-	if err != nil {
-		return 0, err
-	}
-
-	// Write set
-	err = binary.Write(stream, binaryOrder, b.set)
-	return int64(b.BinaryStorageSize()), err
-}
-
-// ReadFrom reads a BitSet from a stream written using WriteTo
-func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) {
-	var length uint64
-
-	// Read length first
-	err := binary.Read(stream, binaryOrder, &length)
-	if err != nil {
-		return 0, err
-	}
-	newset := New(uint(length))
-
-	if uint64(newset.length) != length {
-		return 0, errors.New("unmarshalling error: type mismatch")
-	}
-
-	// Read remaining bytes as set
-	err = binary.Read(stream, binaryOrder, newset.set)
-	if err != nil {
-		return 0, err
-	}
-
-	*b = *newset
-	return int64(b.BinaryStorageSize()), nil
-}
-
-// MarshalBinary encodes a BitSet into a binary form and returns the result.
-func (b *BitSet) MarshalBinary() ([]byte, error) {
-	var buf bytes.Buffer
-	writer := bufio.NewWriter(&buf)
-
-	_, err := b.WriteTo(writer)
-	if err != nil {
-		return []byte{}, err
-	}
-
-	err = writer.Flush()
-
-	return buf.Bytes(), err
-}
-
-// UnmarshalBinary decodes the binary form generated by MarshalBinary.
-func (b *BitSet) UnmarshalBinary(data []byte) error {
-	buf := bytes.NewReader(data)
-	reader := bufio.NewReader(buf)
-
-	_, err := b.ReadFrom(reader)
-
-	return err
-}
-
-// MarshalJSON marshals a BitSet as a JSON structure
-func (b *BitSet) MarshalJSON() ([]byte, error) {
-	buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize()))
-	_, err := b.WriteTo(buffer)
-	if err != nil {
-		return nil, err
-	}
-
-	// URLEncode all bytes
-	return json.Marshal(base64Encoding.EncodeToString(buffer.Bytes()))
-}
-
-// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON
-func (b *BitSet) UnmarshalJSON(data []byte) error {
-	// Unmarshal as string
-	var s string
-	err := json.Unmarshal(data, &s)
-	if err != nil {
-		return err
-	}
-
-	// URLDecode string
-	buf, err := base64Encoding.DecodeString(s)
-	if err != nil {
-		return err
-	}
-
-	_, err = b.ReadFrom(bytes.NewReader(buf))
-	return err
-}

vendor/github.com/bits-and-blooms/bitset/popcnt.go

@@ -1,53 +0,0 @@
-package bitset
-
-// bit population count, take from
-// https://code.google.com/p/go/issues/detail?id=4988#c11
-// credit: https://code.google.com/u/arnehormann/
-func popcount(x uint64) (n uint64) {
-	x -= (x >> 1) & 0x5555555555555555
-	x = (x>>2)&0x3333333333333333 + x&0x3333333333333333
-	x += x >> 4
-	x &= 0x0f0f0f0f0f0f0f0f
-	x *= 0x0101010101010101
-	return x >> 56
-}
-
-func popcntSliceGo(s []uint64) uint64 {
-	cnt := uint64(0)
-	for _, x := range s {
-		cnt += popcount(x)
-	}
-	return cnt
-}
-
-func popcntMaskSliceGo(s, m []uint64) uint64 {
-	cnt := uint64(0)
-	for i := range s {
-		cnt += popcount(s[i] &^ m[i])
-	}
-	return cnt
-}
-
-func popcntAndSliceGo(s, m []uint64) uint64 {
-	cnt := uint64(0)
-	for i := range s {
-		cnt += popcount(s[i] & m[i])
-	}
-	return cnt
-}
-
-func popcntOrSliceGo(s, m []uint64) uint64 {
-	cnt := uint64(0)
-	for i := range s {
-		cnt += popcount(s[i] | m[i])
-	}
-	return cnt
-}
-
-func popcntXorSliceGo(s, m []uint64) uint64 {
-	cnt := uint64(0)
-	for i := range s {
-		cnt += popcount(s[i] ^ m[i])
-	}
-	return cnt
-}