mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2025-01-31 09:09:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #72 from ceph/devel

Browse Source 
Sync devel branch with uptream
This commit is contained in:
OpenShift Merge Robot 2022-02-07 10:18:20 -05:00 committed by GitHub
commit f01260be04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
63 changed files with 633 additions and 1189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
.mergify.yml
View File

@ -37,7 +37,6 @@ queue_rules:
- "status-success=ci/centos/mini-e2e/k8s-1.23"
- "status-success=ci/centos/upgrade-tests-cephfs"
- "status-success=ci/centos/upgrade-tests-rbd"
- "status-success=DCO"
- and:
- base=release-v3.4
- "status-success=codespell"
@ -55,12 +54,10 @@ queue_rules:
- "status-success=ci/centos/mini-e2e/k8s-1.23"
- "status-success=ci/centos/upgrade-tests-cephfs"
- "status-success=ci/centos/upgrade-tests-rbd"
- "status-success=DCO"
- and:
- base=ci/centos
- "status-success=ci/centos/job-validation"
- "status-success=ci/centos/jjb-validate"
- "status-success=DCO"
pull_request_rules:
- name: remove outdated approvals
@ -110,8 +107,15 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
- name: dismiss review of merged pull request
conditions:
- base~=^(devel)|(release-.+)$
- merged
actions:
dismiss_reviews: {}
- name: automatic merge
conditions:
- label!=DNM
@ -142,8 +146,8 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
- name: automatic merge PR having ready-to-merge label
conditions:
- label!=DNM
@ -173,7 +177,6 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
- name: backport patches to release-v3.5 branch
conditions:
@ -252,7 +255,6 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
- name: remove outdated approvals on ci/centos
conditions:
@ -275,7 +277,6 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
- name: automatic merge PR having ready-to-merge label on ci/centos
conditions:
@ -290,7 +291,6 @@ pull_request_rules:
actions:
queue:
name: default
dismiss_reviews: {}
delete_head_branch: {}
##
## Automatically set/remove labels

2
Makefile
View File

@ -203,6 +203,7 @@ ifeq ($(USE_PULLED_IMAGE),no)
.devel-container-id: GOARCH ?= $(shell go env GOARCH 2>/dev/null)
.devel-container-id: .container-cmd scripts/Dockerfile.devel
[ ! -f .devel-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):devel
$(RM) .devel-container-id
$(CONTAINER_CMD) build $(CPUSET) --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg GOARCH=$(GOARCH) -t $(CSI_IMAGE_NAME):devel -f ./scripts/Dockerfile.devel .
$(CONTAINER_CMD) inspect -f '{{.Id}}' $(CSI_IMAGE_NAME):devel > .devel-container-id
else
@ -216,6 +217,7 @@ ifeq ($(USE_PULLED_IMAGE),no)
# create a (cached) container image with dependencies for testing cephcsi
.test-container-id: .container-cmd build.env scripts/Dockerfile.test
[ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test
$(RM) .test-container-id
$(CONTAINER_CMD) build $(CPUSET) --build-arg GOARCH=$(GOARCH) -t $(CSI_IMAGE_NAME):test -f ./scripts/Dockerfile.test .
$(CONTAINER_CMD) inspect -f '{{.Id}}' $(CSI_IMAGE_NAME):test > .test-container-id
else

4
build.env
View File

@ -27,7 +27,7 @@ GOLANGCI_VERSION=v1.43.0
# external snapshotter version
# Refer: https://github.com/kubernetes-csi/external-snapshotter/releases
SNAPSHOT_VERSION=v4.0.0
SNAPSHOT_VERSION=v5.0.1
# "go test" configuration
# set to stdout or html to enable coverage reporting, disabled by default
@ -49,7 +49,7 @@ ROOK_CEPH_CLUSTER_IMAGE=quay.io/ceph/ceph:v16
# CSI sidecar version
CSI_ATTACHER_VERSION=v3.4.0
CSI_SNAPSHOTTER_VERSION=v4.2.0
CSI_SNAPSHOTTER_VERSION=v5.0.1
CSI_PROVISIONER_VERSION=v3.1.0
CSI_RESIZER_VERSION=v1.4.0
CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.4.0

9
charts/ceph-csi-cephfs/templates/provisioner-clusterrole.yaml
View File

@ -27,16 +27,19 @@ rules:
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update"]
verbs: ["update", "patch"]
{{- if .Values.provisioner.attacher.enabled }}
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]

1
charts/ceph-csi-rbd/README.md
View File

@ -149,7 +149,6 @@ charts and their default values.
| `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `<cluster-ID>` |
| `storageClass.dataPool` | Specifies the erasure coded pool | `""` |
| `storageClass.pool` | Ceph pool into which the RBD image shall be created | `replicapool` |
| `storageClass.thickProvision` | Specifies whether thick provision should be enabled | `false` |
| `storageclass.imageFeatures` | Specifies RBD image features | `layering` |
| `storageclass.tryOtherMounters` | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason | `false` |
| `storageClass.mounter` | Specifies RBD mounter | `""` |

9
charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml
View File

@ -38,16 +38,19 @@ rules:
{{- end }}
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update"]
verbs: ["update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]

1
charts/ceph-csi-rbd/templates/storageclass.yaml
View File

@ -18,7 +18,6 @@ parameters:
clusterID: {{ .Values.storageClass.clusterID }}
pool: {{ .Values.storageClass.pool }}
imageFeatures: {{ .Values.storageClass.imageFeatures }}
thickProvision: {{ .Values.storageClass.thickProvision | quote}}
{{- if .Values.storageClass.tryOtherMounters }}
tryOtherMounters: {{ .Values.storageClass.tryOtherMounters | quote}}
{{- end }}

12
charts/ceph-csi-rbd/values.yaml
View File

@ -279,10 +279,6 @@ storageClass:
# eg: pool: replicapool
pool: replicapool
# Set thickProvision to true if you want RBD images to be fully allocated on
# creation (thin provisioning is the default).
thickProvision: false
# (required) RBD image features, CSI creates image with image-format 2
# CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
# `object-map`, `fast-diff` features. If `journaling` is enabled, must
@ -427,14 +423,6 @@ cephconf: |
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
# adding 'fuse_big_writes = true' option by default to override this limit
# see https://github.com/ceph/ceph-csi/issues/1928
fuse_big_writes = true
#########################################################
# Variables for 'internal' use please use with caution! #
#########################################################

5
deploy/cephcsi/image/Dockerfile
View File

@ -23,6 +23,11 @@ RUN source /build.env && \
# test if the downloaded version of Golang works (different arch?)
RUN ${GOROOT}/bin/go version && ${GOROOT}/bin/go env
# FIXME: Ceph does not need Apache Arrow anymore, some container images may
# still have the repository enabled. Disabling the repository can be removed in
# the future, see https://github.com/ceph/ceph-container/pull/1990 .
RUN dnf config-manager --disable apache-arrow-centos || true
RUN dnf -y install \
librados-devel librbd-devel \
/usr/bin/cc \

2
deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
View File

@ -76,7 +76,7 @@ spec:
- name: socket-dir
mountPath: /csi
- name: csi-snapshotter
image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"

7
deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml
View File

@ -31,9 +31,12 @@ rules:
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
@ -51,7 +54,7 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update"]
verbs: ["update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

9
deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
View File

@ -35,10 +35,13 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots/status"]
verbs: ["get", "list", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
@ -53,7 +56,7 @@ rules:
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents/status"]
verbs: ["update"]
verbs: ["update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get"]

2
deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
View File

@ -67,7 +67,7 @@ spec:
- name: socket-dir
mountPath: /csi
- name: csi-snapshotter
image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.0
image: k8s.gcr.io/sig-storage/csi-snapshotter:v5.0.1
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"

35
docs/deploy-rbd.md
View File

@ -26,8 +26,8 @@ make image-cephcsi
**Available command line arguments:**
| Option | Default value | Description |
| ------------------------ | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| Option | Default value | Description |
| ------------------------ | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--endpoint` | `unix:///tmp/csi.sock` | CSI endpoint, must be a UNIX socket |
| `--csi-addons-endpoint` | `unix:///tmp/csi-addons.sock` | CSI-Addons endpoint, must be a UNIX socket |
| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) |
@ -49,23 +49,22 @@ make image-cephcsi
**Available volume parameters:**
| Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `pool` | yes | Ceph pool into which the RBD image shall be created |
| `dataPool` | no | Ceph pool used for the data of the RBD images. |
| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). |
| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). |
| Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `pool` | yes | Ceph pool into which the RBD image shall be created |
| `dataPool` | no | Ceph pool used for the data of the RBD images. |
| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). |
| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). |
| `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. |
| `tryOtherMounters` | no | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason |
| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects |
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** |
| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases |
| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it (**DEPRECATED**: recommended alternative solution is to use accounting/quotas for created volumes) |
| `tryOtherMounters` | no | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason |
| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects |
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** |
| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)

17
docs/design/proposals/cephfs-snapshot-shallow-ro-vol.md
View File

@ -58,7 +58,7 @@ Key points:
* No actual new subvolumes are created in CephFS.
* The resulting volume is a reference to the source subvolume snapshot. This
reference would be stored in `Volume.volume_context` map. In order to
reference a snapshot, we need subvol name and snapshot name.
reference a snapshot, we need subvolume name and snapshot name.
* Mounting such volume means mounting the respective CephFS subvolume and
exposing the snapshot to workloads.
* Let's call a *shallow read-only volume with a subvolume snapshot as its data
@ -73,7 +73,7 @@ a shallow volume is created, what would happen if:
exists?_
This shouldn't be a problem already. The parent volume has either
`snapshot-retention` subvol feature in which case its snapshots remain
`snapshot-retention` subvolume feature in which case its snapshots remain
available, or if it doesn't have that feature, it will fail to be deleted
because it still has snapshots associated to it.
* _Source snapshot from which the shallow volume originates is removed while
@ -256,16 +256,15 @@ Volume context `Volume.volume_context`:
[`NodeStageVolume`, `NodeUnstageVolume` section](#NodeStageVolume-NodeUnstageVolume)
, snapshots cannot be mounted directly. How do we pass in path to the parent
subvolume?
* a) Path to the snapshot is passed in via `subvolumePath` / `rootPath`,
e.g.
`/volumes/<VOLUME NAME>/<SUBVOLUME NAME>/<UUID>/.snap/<SNAPSHOT NAME>`.
From that we can derive path to the subvolume: it's the parent of `.snap`
* a) Path to the snapshot is passed in via `subvolumePath` / `rootPath`, e.g.
`/volumes/<VOLUME NAME>/<SUBVOLUME NAME>/<UUID>/.snap/<SNAPSHOT NAME>`. From
that we can derive path to the subvolume: it's the parent of `.snap`
directory.
* b) Similar to a), path to the snapshot is passed in via `subvolumePath` /
`rootPath`, but instead of trying to derive the right path we introduce
another volume context parameter containing path to the parent subvolume
explicitly.
* c) `subvolumePath` / `rootPath` contains path to the parent subvolume and
we introduce another volume context parameter containing name of the
snapshot. Path to the snapshot is then formed by appending
* c) `subvolumePath` / `rootPath` contains path to the parent subvolume and we
introduce another volume context parameter containing name of the snapshot.
Path to the snapshot is then formed by appending
`/.snap/<SNAPSHOT NAME>` to the subvolume path.

12
docs/design/proposals/encryped-with-keyprotect.md → docs/design/proposals/encryption-with-keyprotect.md
View File

@ -16,7 +16,7 @@ service from the CSI driver and to make use of the encryption operations:
The Key Protect/HPCS connection URL.
* IBM_KP_TOKEN_URL
The Token Authenticaltion URL of KeyProtect/HPCS service.
The Token Authentication URL of KeyProtect/HPCS service.
* KMS_SERVICE_NAME
A unique name for the key management service within the project.
@ -31,14 +31,14 @@ Ex: 06x6DbTkVQ-qCRmq9cK-p9xOQpU2UwJMcdjnIDdr0g2R
Ex: c7a9aa91-5cb5-48da-a821-e85c27b99d92
* IBM_KP_REGION
Region of the key protect service, ex: us-south-2
Region of the Key Protect service, ex: us-south-2
```
### Values provided in the connection Secret
Considering `SERVICE_API_KEY` and `CUSTOMER_ROOT_KEY` are sensitive information,
those will be provided as a Kubernetes Secret to the CSI driver. The Ceph CSI
KMS plugin interface for the key protect will read the Secret name from the kms
KMS plugin interface for the Key Protect will read the Secret name from the kms
ConfigMap and fetch these values. `SESSION_TOKEN and CRK_ARN` values can also be
provided by the user as part of the Secret if needed. How-ever these values are
considered to be optional.
@ -56,7 +56,7 @@ config map to `KMS_SERVICE_NAME`.
## Volume Encrypt or Decrypt Operation
The IBM Key protect server's `wrap` and `unwrap` functionalities will be used by
The IBM Key Protect server's `wrap` and `unwrap` functionalities will be used by
the Ceph CSI driver to achieve encryption and decryption of volumes. The DEK can
be wrapped with the help of Customer Root Key (CRK) and can be used for LUKS
operation. The wrapped cipher blob will be stored inside the image metadata ( as
@ -66,9 +66,9 @@ with the help of cipher blob and Key Protect server
## Integration APIS
[Key Protect Go Client](https://github.com/IBM/keyprotect-go-client) provide the
client SDK to interact with the Key Protect server and perform key protect
client SDK to interact with the Key Protect server and perform Key Protect
operations.
## Additional Reference
[Key Protect Doc](https://cloud.ibm.com/docs/key-protect)
[Key Protect Doc](https://cloud.ibm.com/docs/key-protect)

2
docs/snap-clone.md
View File

@ -33,7 +33,7 @@
`SNAPSHOT_VERSION` variable, for example:
```console
SNAPSHOT_VERSION="v4.0.0" ./scripts/install-snapshot.sh install
SNAPSHOT_VERSION="v5.0.1" ./scripts/install-snapshot.sh install
```
- In the future, you can choose to cleanup by running

193
e2e/rbd.go
View File

@ -1265,11 +1265,11 @@ var _ = Describe("RBD", func() {
var runningAttachCmd string
runningAttachCmd, stdErr, err = execCommandInContainer(
f,
"ps -eo 'cmd' | grep [r]bd-nbd",
"pstree --arguments | grep [r]bd-nbd",
cephCSINamespace,
"csi-rbdplugin",
&opt)
// if the rbd-nbd process is not running the ps | grep command
// if the rbd-nbd process is not running the 'grep' command
// will return with exit code 1
if err != nil {
if strings.Contains(err.Error(), "command terminated with exit code 1") {
@ -1747,38 +1747,6 @@ var _ = Describe("RBD", func() {
f)
})
By("create a thick-provisioned PVC-PVC clone and bind it to an app", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{
"thickProvision": "true",
}, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
validatePVCClone(1,
pvcPath,
appPath,
pvcSmartClonePath,
appSmartClonePath,
noDataPool,
noKMS,
isThickPVC,
f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create an encrypted PVC snapshot and restore it for an app with VaultKMS", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -1863,75 +1831,6 @@ var _ = Describe("RBD", func() {
}
})
By("Validate thick PVC restore from vaultKMS to userSecretsMetadataKMS", func() {
restoreSCName := "restore-sc"
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
scOpts := map[string]string{
"encrypted": "true",
"encryptionKMSID": "vault-test",
"thickProvision": "true",
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
scOpts = map[string]string{
"encrypted": "true",
"encryptionKMSID": "user-secrets-metadata-test",
"thickProvision": "true",
}
err = createRBDStorageClass(f.ClientSet, f, restoreSCName, nil, scOpts, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
// PVC creation namespace where secret will be created
namespace := f.UniqueName
// create user Secret
err = retryKubectlFile(namespace, kubectlCreate, vaultExamplePath+vaultUserSecret, deployTimeout)
if err != nil {
e2elog.Failf("failed to create user Secret: %v", err)
}
validatePVCSnapshot(1,
pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath,
vaultKMS, secretsMetadataKMS,
restoreSCName, noDataPool, f)
// delete user secret
err = retryKubectlFile(namespace,
kubectlDelete,
vaultExamplePath+vaultUserSecret,
deployTimeout,
"--ignore-not-found=true")
if err != nil {
e2elog.Failf("failed to delete user Secret: %v", err)
}
err = retryKubectlArgs(cephCSINamespace, kubectlDelete, deployTimeout, "storageclass", restoreSCName)
if err != nil {
e2elog.Failf("failed to delete storageclass %q: %v", restoreSCName, err)
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create an encrypted PVC-PVC clone and bind it to an app", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -3180,47 +3079,6 @@ var _ = Describe("RBD", func() {
validateRBDImageCount(f, 0, defaultRBDPool)
})
By("create a thick-provisioned PVC", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{
"thickProvision": "true",
}, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
pvc, err := loadPVC(rawPvcPath)
if err != nil {
e2elog.Failf("failed to load PVC:: %v", err)
}
pvcSizes := []string{
// original value from the yaml file (100MB)
"100Mi",
// half the size (50MB), is not stripe-size roundable
"50Mi",
}
for _, pvcSize := range pvcSizes {
err = validateThickPVC(f, pvc, pvcSize)
if err != nil {
e2elog.Failf("validating thick-provisioning failed: %v", err)
}
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create a PVC and Bind it to an app for mapped rbd image with options", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
@ -3270,53 +3128,6 @@ var _ = Describe("RBD", func() {
}
})
By("validate the functionality of controller with encryption and thick-provisioning", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass : %v", err)
}
scParams := map[string]string{
"encrypted": "true",
"encryptionKMSID": "user-secrets-metadata-test",
"thickProvision": "true",
}
// PVC creation namespace where secret will be created
namespace := f.UniqueName
// create user Secret
err = retryKubectlFile(namespace, kubectlCreate, vaultExamplePath+vaultUserSecret, deployTimeout)
if err != nil {
e2elog.Failf("failed to create user Secret: %v", err)
}
err = validateController(f,
pvcPath, appPath, rbdExamplePath+"storageclass.yaml",
nil,
scParams)
if err != nil {
e2elog.Failf("failed to validate controller : %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
// delete user secret
err = retryKubectlFile(namespace,
kubectlDelete,
vaultExamplePath+vaultUserSecret,
deployTimeout,
"--ignore-not-found=true")
if err != nil {
e2elog.Failf("failed to delete user Secret: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass : %v", err)
}
})
By("validate image deletion when it is moved to trash", func() {
// make sure pool is empty
validateRBDImageCount(f, 0, defaultRBDPool)

104
e2e/rbd_helper.go
View File

@ -31,7 +31,6 @@ import (
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
v1 "k8s.io/api/core/v1"
scv1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
@ -39,11 +38,6 @@ import (
e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)
const (
// image metadata key for thick-provisioning.
thickProvisionMetaKey = "rbd.csi.ceph.com/thick-provisioned"
)
// nolint:gomnd // numbers specify Kernel versions.
var nbdResizeSupport = []util.KernelVersion{
{
@ -529,39 +523,6 @@ func isEncryptedPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *
return validateEncryptedImage(f, rbdImageSpec, imageData.pvName, app.Name)
}
func isThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
du, err := getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize == 0 || du.UsedSize != du.ProvisionedSize {
return fmt.Errorf("backing RBD image is not thick-provisioned (%d/%d)", du.UsedSize, du.ProvisionedSize)
}
err = validateThickImageMetadata(f, pvc, thickProvisionMetaKey)
if err != nil {
return fmt.Errorf("failed to validate thick image: %w", err)
}
return nil
}
// validateThickImage check thick metadata is set on the the image.
func validateThickImageMetadata(f *framework.Framework, pvc *v1.PersistentVolumeClaim, key string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
thickState, err := getImageMeta(rbdImageSpec, key, f)
if err != nil {
return err
}
if thickState == "" {
return fmt.Errorf("image metadata is set for %s", rbdImageSpec)
}
return nil
}
// validateEncryptedImage verifies that the RBD image is encrypted. The
// following checks are performed:
// - Metadata of the image should be set with the encryption state;
@ -630,6 +591,7 @@ func deleteBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim
return err
}
//nolint:unused // required for reclaimspace e2e.
// rbdDuImage contains the disk-usage statistics of an RBD image.
type rbdDuImage struct {
Name string `json:"name"`
@ -637,11 +599,13 @@ type rbdDuImage struct {
UsedSize uint64 `json:"used_size"`
}
//nolint:unused // required for reclaimspace e2e.
// rbdDuImageList contains the list of images returned by 'rbd du'.
type rbdDuImageList struct {
Images []*rbdDuImage `json:"images"`
}
//nolint:deadcode,unused // required for reclaimspace e2e.
// getRbdDu runs 'rbd du' on the RBD image and returns a rbdDuImage struct with
// the result.
func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImage, error) {
@ -672,6 +636,7 @@ func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImag
return nil, fmt.Errorf("image %s not found", imageData.imageName)
}
//nolint:deadcode,unused // required for reclaimspace e2e.
// sparsifyBackingRBDImage runs `rbd sparsify` on the RBD image. Once done, all
// data blocks that contain zeros are discarded/trimmed/unmapped and do not
// take up any space anymore. This can be used to verify that an empty, but
@ -903,67 +868,6 @@ func deletePVCCSIJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeC
return nil
}
func validateThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, size string) error {
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(size)
err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to create PVC: %w", err)
}
validateRBDImageCount(f, 1, defaultRBDPool)
err = validateThickImageMetadata(f, pvc, thickProvisionMetaKey)
if err != nil {
return fmt.Errorf("failed to validate thick image: %w", err)
}
// nothing has been written, but the image should be allocated
du, err := getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize == 0 || du.UsedSize != du.ProvisionedSize {
return fmt.Errorf("backing RBD image is not thick-provisioned (%d/%d)", du.UsedSize, du.ProvisionedSize)
}
// expanding the PVC should thick-allocate the expansion
// nolint:gomnd // we want 2x the size so that extending is done
newSize := du.ProvisionedSize * 2
err = expandPVCSize(f.ClientSet, pvc, fmt.Sprintf("%d", newSize), deployTimeout)
if err != nil {
return fmt.Errorf("failed to expand PVC: %w", err)
}
// after expansion, the updated 'du' should be larger
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize != newSize {
return fmt.Errorf("backing RBD image is not extended thick-provisioned (%d/%d)", du.UsedSize, newSize)
}
// thick provisioning allows for sparsifying
err = sparsifyBackingRBDImage(f, pvc)
if err != nil {
return fmt.Errorf("failed to sparsify RBD image: %w", err)
}
// after sparsifying the image should not have any allocations
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("backing RBD image is not thick-provisioned: %w", err)
} else if du.UsedSize != 0 {
return fmt.Errorf("backing RBD image was not sparsified (%d bytes allocated)", du.UsedSize)
}
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to delete PVC:: %w", err)
}
validateRBDImageCount(f, 0, defaultRBDPool)
return nil
}
// trashInfo contains the image details in trash.
type trashInfo struct {
Name string `json:"name"`

7
examples/ceph-conf.yaml
View File

@ -15,13 +15,6 @@ data:
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
# adding 'fuse_big_writes = true' option by default to override this limit
# see https://github.com/ceph/ceph-csi/issues/1928
fuse_big_writes = true
# keyring is a required key and its value should be empty
keyring: |
metadata:

4
examples/kms/vault/csi-kms-connection-details.yaml
View File

@ -57,7 +57,7 @@ data:
aws-metadata-test: |-
{
"KMS_PROVIDER": "aws-metadata",
"IBM_KP_SECRET_NAME": "ceph-csi-aws-credentials",
"KMS_SECRET_NAME": "ceph-csi-aws-credentials",
"AWS_REGION": "us-west-2"
}
ibmkeyprotect-test: |-
@ -66,7 +66,7 @@ data:
"IBM_KP_SECRET_NAME": "ceph-csi-kp-credentials",
"IBM_KP_SERVICE_INSTANCE_ID": "7abef064-01dd-4237-9ea5-8b3890970be3",
"IBM_KP_BASE_URL": "https://us-south.kms.cloud.ibm.com",
"IBM_KP_TOKEN_URL": ""https://iam.cloud.ibm.com/oidc/token",
"IBM_KP_TOKEN_URL": "https://iam.cloud.ibm.com/oidc/token",
"IBM_KP_REGION": "us-south-2",
}
metadata:

4
examples/rbd/storageclass.yaml
View File

@ -29,10 +29,6 @@ parameters:
# eg: pool: rbdpool
pool: <rbd-pool-name>
# Deprecated: Set thickProvision to true if you want RBD images to be fully
# allocated on creation (thin provisioning is the default).
# thickProvision: "false"
# (required) RBD image features, CSI creates image with image-format 2
# CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
# `object-map`, `fast-diff` features. If `journaling` is enabled, must

4
go.mod
View File

@ -18,9 +18,9 @@ require (
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
github.com/libopenstorage/secrets v0.0.0-20210908194121-a1d19aa9713a
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.17.0
github.com/onsi/gomega v1.18.0
github.com/pborman/uuid v1.2.1
github.com/prometheus/client_golang v1.12.0
github.com/prometheus/client_golang v1.12.1
github.com/stretchr/testify v1.7.0
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9

11
go.sum
View File

@ -461,6 +461,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -851,14 +852,17 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.18.0 h1:ngbYoRctxjl8SiF7XgP0NxBFbfHcg3wfHMMaFHWwMTM=
github.com/onsi/gomega v1.18.0/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
@ -932,8 +936,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg=
github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@ -1386,6 +1390,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=

62
internal/cephfs/core/filesystem.go
View File

@ -25,76 +25,102 @@ import (
"github.com/ceph/ceph-csi/internal/util/log"
)
func (vo *VolumeOptions) getFscID(ctx context.Context) (int64, error) {
fsa, err := vo.conn.GetFSAdmin()
// FileSystem is the interface that holds the signature of filesystem methods
// that interacts with CephFS filesystem API's.
type FileSystem interface {
// GetFscID returns the ID of the filesystem with the given name.
GetFscID(context.Context, string) (int64, error)
// GetMetadataPool returns the metadata pool name of the filesystem with the given name.
GetMetadataPool(context.Context, string) (string, error)
// GetFsName returns the name of the filesystem with the given ID.
GetFsName(context.Context, int64) (string, error)
}
// fileSystem is the implementation of FileSystem interface.
type fileSystem struct {
conn *util.ClusterConnection
}
// NewFileSystem returns a new instance of fileSystem.
func NewFileSystem(conn *util.ClusterConnection) FileSystem {
return &fileSystem{
conn: conn,
}
}
// GetFscID returns the ID of the filesystem with the given name.
func (f *fileSystem) GetFscID(ctx context.Context, fsName string) (int64, error) {
fsa, err := f.conn.GetFSAdmin()
if err != nil {
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem ID for %s:", vo.FsName, err)
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem ID for %s: %s", fsName, err)
return 0, err
}
volumes, err := fsa.EnumerateVolumes()
if err != nil {
log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem ID for %s:", vo.FsName, err)
log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem ID for %s: %s", fsName, err)
return 0, err
}
for _, vol := range volumes {
if vol.Name == vo.FsName {
if vol.Name == fsName {
return vol.ID, nil
}
}
log.ErrorLog(ctx, "failed to list volume %s", vo.FsName)
log.ErrorLog(ctx, "failed to list volume %s", fsName)
return 0, cerrors.ErrVolumeNotFound
}
func (vo *VolumeOptions) getMetadataPool(ctx context.Context) (string, error) {
fsa, err := vo.conn.GetFSAdmin()
// GetMetadataPool returns the metadata pool name of the filesystem with the given name.
func (f *fileSystem) GetMetadataPool(ctx context.Context, fsName string) (string, error) {
fsa, err := f.conn.GetFSAdmin()
if err != nil {
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err)
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s: %s", fsName, err)
return "", err
}
fsPoolInfos, err := fsa.ListFileSystems()
if err != nil {
log.ErrorLog(ctx, "could not list filesystems, can not fetch metadata pool for %s:", vo.FsName, err)
log.ErrorLog(ctx, "could not list filesystems, can not fetch metadata pool for %s: %s", fsName, err)
return "", err
}
for _, fspi := range fsPoolInfos {
if fspi.Name == vo.FsName {
if fspi.Name == fsName {
return fspi.MetadataPool, nil
}
}
return "", fmt.Errorf("%w: could not find metadata pool for %s", util.ErrPoolNotFound, vo.FsName)
return "", fmt.Errorf("%w: could not find metadata pool for %s", util.ErrPoolNotFound, fsName)
}
func (vo *VolumeOptions) getFsName(ctx context.Context) (string, error) {
fsa, err := vo.conn.GetFSAdmin()
// GetFsName returns the name of the filesystem with the given ID.
func (f *fileSystem) GetFsName(ctx context.Context, fscID int64) (string, error) {
fsa, err := f.conn.GetFSAdmin()
if err != nil {
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem name for ID %d:", vo.FscID, err)
log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem name for ID %d: %s", fscID, err)
return "", err
}
volumes, err := fsa.EnumerateVolumes()
if err != nil {
log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem name for ID %d:", vo.FscID, err)
log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem name for ID %d: %s", fscID, err)
return "", err
}
for _, vol := range volumes {
if vol.ID == vo.FscID {
if vol.ID == fscID {
return vol.Name, nil
}
}
return "", fmt.Errorf("%w: fscID (%d) not found in Ceph cluster", util.ErrPoolNotFound, vo.FscID)
return "", fmt.Errorf("%w: fscID (%d) not found in Ceph cluster", util.ErrPoolNotFound, fscID)
}

15
internal/cephfs/core/volumeoptions.go
View File

@ -230,12 +230,13 @@ func NewVolumeOptions(ctx context.Context, requestName string, req *csi.CreateVo
return nil, err
}
opts.FscID, err = opts.getFscID(ctx)
fs := NewFileSystem(opts.conn)
opts.FscID, err = fs.GetFscID(ctx, opts.FsName)
if err != nil {
return nil, err
}
opts.MetadataPool, err = opts.getMetadataPool(ctx)
opts.MetadataPool, err = fs.GetMetadataPool(ctx, opts.FsName)
if err != nil {
return nil, err
}
@ -308,12 +309,13 @@ func NewVolumeOptionsFromVolID(
}
}()
volOptions.FsName, err = volOptions.getFsName(ctx)
fs := NewFileSystem(volOptions.conn)
volOptions.FsName, err = fs.GetFsName(ctx, volOptions.FscID)
if err != nil {
return nil, nil, err
}
volOptions.MetadataPool, err = volOptions.getMetadataPool(ctx)
volOptions.MetadataPool, err = fs.GetMetadataPool(ctx, volOptions.FsName)
if err != nil {
return nil, nil, err
}
@ -549,12 +551,13 @@ func NewSnapshotOptionsFromID(
}
}()
volOptions.FsName, err = volOptions.getFsName(ctx)
fs := NewFileSystem(volOptions.conn)
volOptions.FsName, err = fs.GetFsName(ctx, volOptions.FscID)
if err != nil {
return &volOptions, nil, &sid, err
}
volOptions.MetadataPool, err = volOptions.getMetadataPool(ctx)
volOptions.MetadataPool, err = fs.GetMetadataPool(ctx, volOptions.FsName)
if err != nil {
return &volOptions, nil, &sid, err
}

16
internal/kms/aws_metadata.go
View File

@ -65,7 +65,7 @@ var _ = RegisterProvider(Provider{
Initializer: initAWSMetadataKMS,
})
type AWSMetadataKMS struct {
type awsMetadataKMS struct {
// basic options to get the secret
namespace string
secretName string
@ -79,7 +79,7 @@ type AWSMetadataKMS struct {
}
func initAWSMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
kms := &AWSMetadataKMS{
kms := &awsMetadataKMS{
namespace: args.Namespace,
}
@ -124,7 +124,7 @@ func initAWSMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
return kms, nil
}
func (kms *AWSMetadataKMS) getSecrets() (map[string]interface{}, error) {
func (kms *awsMetadataKMS) getSecrets() (map[string]interface{}, error) {
c, err := k8s.NewK8sClient()
if err != nil {
return nil, fmt.Errorf("failed to connect to Kubernetes to "+
@ -153,18 +153,18 @@ func (kms *AWSMetadataKMS) getSecrets() (map[string]interface{}, error) {
return config, nil
}
func (kms *AWSMetadataKMS) Destroy() {
func (kms *awsMetadataKMS) Destroy() {
// Nothing to do.
}
// RequiresDEKStore indicates that the DEKs should get stored in the metadata
// of the volumes. This Amazon KMS provider does not support storing DEKs in
// AWS as that adds additional costs.
func (kms *AWSMetadataKMS) RequiresDEKStore() DEKStoreType {
func (kms *awsMetadataKMS) RequiresDEKStore() DEKStoreType {
return DEKStoreMetadata
}
func (kms *AWSMetadataKMS) getService() (*awsKMS.KMS, error) {
func (kms *awsMetadataKMS) getService() (*awsKMS.KMS, error) {
creds := awsCreds.NewStaticCredentials(kms.accessKey,
kms.secretAccessKey, kms.sessionToken)
@ -183,7 +183,7 @@ func (kms *AWSMetadataKMS) getService() (*awsKMS.KMS, error) {
}
// EncryptDEK uses the Amazon KMS and the configured CMK to encrypt the DEK.
func (kms *AWSMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
func (kms *awsMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
svc, err := kms.getService()
if err != nil {
return "", fmt.Errorf("could not get KMS service: %w", err)
@ -206,7 +206,7 @@ func (kms *AWSMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error)
}
// DecryptDEK uses the Amazon KMS and the configured CMK to decrypt the DEK.
func (kms *AWSMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
func (kms *awsMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
svc, err := kms.getService()
if err != nil {
return "", fmt.Errorf("could not get KMS service: %w", err)

18
internal/kms/keyprotect.go
View File

@ -79,7 +79,7 @@ func initKeyProtectKMSOld(args ProviderInitArgs) (EncryptionKMS, error) {
}
// KeyProtectKMS store the KMS connection information retrieved from the kms configmap.
type KeyProtectKMS struct {
type keyProtectKMS struct {
// basic options to get the secret
namespace string
secretName string
@ -97,7 +97,7 @@ type KeyProtectKMS struct {
}
func initKeyProtectKMS(args ProviderInitArgs) (EncryptionKMS, error) {
kms := &KeyProtectKMS{
kms := &keyProtectKMS{
namespace: args.Namespace,
}
// required options for further configuration (getting secrets)
@ -164,7 +164,7 @@ func initKeyProtectKMS(args ProviderInitArgs) (EncryptionKMS, error) {
return kms, nil
}
func (kms *KeyProtectKMS) getSecrets() (map[string]interface{}, error) {
func (kms *keyProtectKMS) getSecrets() (map[string]interface{}, error) {
c, err := k8s.NewK8sClient()
if err != nil {
return nil, fmt.Errorf("failed to connect to Kubernetes to "+
@ -193,16 +193,16 @@ func (kms *KeyProtectKMS) getSecrets() (map[string]interface{}, error) {
return config, nil
}
func (kms *KeyProtectKMS) Destroy() {
func (kms *keyProtectKMS) Destroy() {
// Nothing to do.
}
func (kms *KeyProtectKMS) RequiresDEKStore() DEKStoreType {
func (kms *keyProtectKMS) RequiresDEKStore() DEKStoreType {
return DEKStoreMetadata
}
func (kms *KeyProtectKMS) getService() error {
// Use Service API Key and KeyProtect Service Instance ID to create a ClientConfig
func (kms *keyProtectKMS) getService() error {
// Use your Service API Key and your KeyProtect Service Instance ID to create a ClientConfig
cc := kp.ClientConfig{
BaseURL: kms.baseURL,
TokenURL: kms.tokenURL,
@ -221,7 +221,7 @@ func (kms *KeyProtectKMS) getService() error {
}
// EncryptDEK uses the KeyProtect KMS and the configured CRK to encrypt the DEK.
func (kms *KeyProtectKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
func (kms *keyProtectKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
if err := kms.getService(); err != nil {
return "", fmt.Errorf("could not get KMS service: %w", err)
}
@ -240,7 +240,7 @@ func (kms *KeyProtectKMS) EncryptDEK(volumeID, plainDEK string) (string, error)
}
// DecryptDEK uses the Key protect KMS and the configured CRK to decrypt the DEK.
func (kms *KeyProtectKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
func (kms *keyProtectKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
if err := kms.getService(); err != nil {
return "", fmt.Errorf("could not get KMS service: %w", err)
}

10
internal/kms/kms.go
View File

@ -359,20 +359,20 @@ type DEKStore interface {
RemoveDEK(volumeID string) error
}
// IntegratedDEK is a DEKStore that can not be configured. Either the KMS does
// integratedDEK is a DEKStore that can not be configured. Either the KMS does
// not use a DEK, or the DEK is stored in the KMS without additional
// configuration options.
type IntegratedDEK struct{}
type integratedDEK struct{}
func (i IntegratedDEK) RequiresDEKStore() DEKStoreType {
func (i integratedDEK) RequiresDEKStore() DEKStoreType {
return DEKStoreIntegrated
}
func (i IntegratedDEK) EncryptDEK(volumeID, plainDEK string) (string, error) {
func (i integratedDEK) EncryptDEK(volumeID, plainDEK string) (string, error) {
return plainDEK, nil
}
func (i IntegratedDEK) DecryptDEK(volumeID, encyptedDEK string) (string, error) {
func (i integratedDEK) DecryptDEK(volumeID, encyptedDEK string) (string, error) {
return encyptedDEK, nil
}

58
internal/kms/secretskms.go
View File

@ -36,7 +36,7 @@ const (
// Encryption passphrase location in K8s secrets.
encryptionPassphraseKey = "encryptionPassphrase"
// kmsTypeSecretsMetadata is the SecretsKMS with per-volume encryption,
// kmsTypeSecretsMetadata is the secretKMS with per-volume encryption,
// where the DEK is stored in the metadata of the volume itself.
kmsTypeSecretsMetadata = "metadata"
@ -48,9 +48,9 @@ const (
metadataSecretNamespaceKey = "secretNamespace"
)
// SecretsKMS is default KMS implementation that means no KMS is in use.
type SecretsKMS struct {
IntegratedDEK
// secretsKMS is default KMS implementation that means no KMS is in use.
type secretsKMS struct {
integratedDEK
passphrase string
}
@ -60,7 +60,7 @@ var _ = RegisterProvider(Provider{
Initializer: newSecretsKMS,
})
// newSecretsKMS initializes a SecretsKMS that uses the passphrase from the
// newSecretsKMS initializes a secretsKMS that uses the passphrase from the
// secret that is configured for the StorageClass. This KMS provider uses a
// single (LUKS) passhprase for all volumes.
func newSecretsKMS(args ProviderInitArgs) (EncryptionKMS, error) {
@ -69,35 +69,35 @@ func newSecretsKMS(args ProviderInitArgs) (EncryptionKMS, error) {
return nil, errors.New("missing encryption passphrase in secrets")
}
return SecretsKMS{passphrase: passphraseValue}, nil
return secretsKMS{passphrase: passphraseValue}, nil
}
// Destroy frees all used resources.
func (kms SecretsKMS) Destroy() {
func (kms secretsKMS) Destroy() {
// nothing to do
}
// FetchDEK returns passphrase from Kubernetes secrets.
func (kms SecretsKMS) FetchDEK(key string) (string, error) {
func (kms secretsKMS) FetchDEK(key string) (string, error) {
return kms.passphrase, nil
}
// StoreDEK does nothing, as there is no passphrase per key (volume), so
// no need to store is anywhere.
func (kms SecretsKMS) StoreDEK(key, value string) error {
func (kms secretsKMS) StoreDEK(key, value string) error {
return nil
}
// RemoveDEK is doing nothing as no new passphrases are saved with
// SecretsKMS.
func (kms SecretsKMS) RemoveDEK(key string) error {
// secretsKMS.
func (kms secretsKMS) RemoveDEK(key string) error {
return nil
}
// SecretsMetadataKMS is a KMS based on the SecretsKMS, but stores the
// secretsMetadataKMS is a KMS based on the secretKMS, but stores the
// Data-Encryption-Key (DEK) in the metadata of the volume.
type SecretsMetadataKMS struct {
SecretsKMS
type secretsMetadataKMS struct {
secretsKMS
}
var _ = RegisterProvider(Provider{
@ -105,12 +105,12 @@ var _ = RegisterProvider(Provider{
Initializer: initSecretsMetadataKMS,
})
// initSecretsMetadataKMS initializes a SecretsMetadataKMS that wraps a SecretsKMS,
// initSecretsMetadataKMS initializes a secretsMetadataKMS that wraps a secretKMS,
// so that the passphrase from the user provided or StorageClass secrets can be used
// for encrypting/decrypting DEKs that are stored in a detached DEKStore.
func initSecretsMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
var (
smKMS SecretsMetadataKMS
smKMS secretsMetadataKMS
encryptionPassphrase string
ok bool
err error
@ -130,13 +130,13 @@ func initSecretsMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) {
"missing %q in storageclass secret", encryptionPassphraseKey)
}
}
smKMS.SecretsKMS = SecretsKMS{passphrase: encryptionPassphrase}
smKMS.secretsKMS = secretsKMS{passphrase: encryptionPassphrase}
return smKMS, nil
}
// fetchEncryptionPassphrase fetches encryptionPassphrase from user provided secret.
func (kms SecretsMetadataKMS) fetchEncryptionPassphrase(
func (kms secretsMetadataKMS) fetchEncryptionPassphrase(
config map[string]interface{},
defaultNamespace string) (string, error) {
var (
@ -182,11 +182,11 @@ func (kms SecretsMetadataKMS) fetchEncryptionPassphrase(
}
// Destroy frees all used resources.
func (kms SecretsMetadataKMS) Destroy() {
kms.SecretsKMS.Destroy()
func (kms secretsMetadataKMS) Destroy() {
kms.secretsKMS.Destroy()
}
func (kms SecretsMetadataKMS) RequiresDEKStore() DEKStoreType {
func (kms secretsMetadataKMS) RequiresDEKStore() DEKStoreType {
return DEKStoreMetadata
}
@ -202,12 +202,12 @@ type encryptedMetedataDEK struct {
}
// EncryptDEK encrypts the plainDEK with a key derived from the passphrase from
// the SecretsKMS and the volumeID.
// the secretsKMS and the volumeID.
// The resulting encryptedDEK contains a JSON with the encrypted DEK and the
// nonce that was used for encrypting.
func (kms SecretsMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
// use the passphrase from the SecretsKMS
passphrase, err := kms.SecretsKMS.FetchDEK(volumeID)
func (kms secretsMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, error) {
// use the passphrase from the secretKMS
passphrase, err := kms.secretsKMS.FetchDEK(volumeID)
if err != nil {
return "", fmt.Errorf("failed to get passphrase: %w", err)
}
@ -234,10 +234,10 @@ func (kms SecretsMetadataKMS) EncryptDEK(volumeID, plainDEK string) (string, err
}
// DecryptDEK takes the JSON formatted `encryptedMetadataDEK` contents, and it
// fetches SecretsKMS passphrase to decrypt the DEK.
func (kms SecretsMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
// use the passphrase from the SecretsKMS
passphrase, err := kms.SecretsKMS.FetchDEK(volumeID)
// fetches secretKMS passphrase to decrypt the DEK.
func (kms secretsMetadataKMS) DecryptDEK(volumeID, encryptedDEK string) (string, error) {
// use the passphrase from the secretKMS
passphrase, err := kms.secretsKMS.FetchDEK(volumeID)
if err != nil {
return "", fmt.Errorf("failed to get passphrase: %w", err)
}

18
internal/kms/vault.go
View File

@ -87,9 +87,9 @@ type vaultConnection struct {
vaultDestroyKeys bool
}
type VaultKMS struct {
type vaultKMS struct {
vaultConnection
IntegratedDEK
integratedDEK
// vaultPassphrasePath (VPP) used to be added before the "key" of the
// secret (like /v1/secret/data/<VPP>/key)
@ -192,6 +192,11 @@ func (vc *vaultConnection) initConnection(config map[string]interface{}) error {
if errors.Is(err, errConfigOptionInvalid) {
return err
}
// set the option if the value was not invalid
if firstInit || !errors.Is(err, errConfigOptionMissing) {
keyContext[loss.KeyVaultNamespace] = vaultNamespace
}
vaultAuthNamespace := ""
err = setConfigString(&vaultAuthNamespace, config, "vaultAuthNamespace")
if errors.Is(err, errConfigOptionInvalid) {
@ -205,7 +210,6 @@ func (vc *vaultConnection) initConnection(config map[string]interface{}) error {
// set the option if the value was not invalid
if firstInit || !errors.Is(err, errConfigOptionMissing) {
vaultConfig[api.EnvVaultNamespace] = vaultAuthNamespace
keyContext[loss.KeyVaultNamespace] = vaultNamespace
}
verifyCA := strconv.FormatBool(vaultDefaultCAVerify) // optional
@ -329,7 +333,7 @@ var _ = RegisterProvider(Provider{
// InitVaultKMS returns an interface to HashiCorp Vault KMS.
func initVaultKMS(args ProviderInitArgs) (EncryptionKMS, error) {
kms := &VaultKMS{}
kms := &vaultKMS{}
err := kms.initConnection(args.Config)
if err != nil {
return nil, fmt.Errorf("failed to initialize Vault connection: %w", err)
@ -392,7 +396,7 @@ func initVaultKMS(args ProviderInitArgs) (EncryptionKMS, error) {
// FetchDEK returns passphrase from Vault. The passphrase is stored in a
// data.data.passphrase structure.
func (kms *VaultKMS) FetchDEK(key string) (string, error) {
func (kms *vaultKMS) FetchDEK(key string) (string, error) {
s, err := kms.secrets.GetSecret(filepath.Join(kms.vaultPassphrasePath, key), kms.keyContext)
if err != nil {
return "", err
@ -411,7 +415,7 @@ func (kms *VaultKMS) FetchDEK(key string) (string, error) {
}
// StoreDEK saves new passphrase in Vault.
func (kms *VaultKMS) StoreDEK(key, value string) error {
func (kms *vaultKMS) StoreDEK(key, value string) error {
data := map[string]interface{}{
"data": map[string]string{
"passphrase": value,
@ -428,7 +432,7 @@ func (kms *VaultKMS) StoreDEK(key, value string) error {
}
// RemoveDEK deletes passphrase from Vault.
func (kms *VaultKMS) RemoveDEK(key string) error {
func (kms *vaultKMS) RemoveDEK(key string) error {
pathKey := filepath.Join(kms.vaultPassphrasePath, key)
err := kms.secrets.DeleteSecret(pathKey, kms.getDeleteKeyContext())
if err != nil {

24
internal/kms/vault_sa.go
View File

@ -67,7 +67,7 @@ Example JSON structure in the KMS config is,
...
}.
*/
type VaultTenantSA struct {
type vaultTenantSA struct {
vaultTenantConnection
// tenantSAName is the name of the ServiceAccount in the Tenants Kubernetes Namespace
@ -97,7 +97,7 @@ func initVaultTenantSA(args ProviderInitArgs) (EncryptionKMS, error) {
}
}
kms := &VaultTenantSA{}
kms := &vaultTenantSA{}
kms.vaultTenantConnection.init()
kms.tenantConfigOptionFilter = isTenantSAConfigOption
@ -150,7 +150,7 @@ func initVaultTenantSA(args ProviderInitArgs) (EncryptionKMS, error) {
// Destroy removes the temporary stored token from the ServiceAccount and
// destroys the vaultTenantConnection object.
func (kms *VaultTenantSA) Destroy() {
func (kms *vaultTenantSA) Destroy() {
if kms.saTokenDir != "" {
_ = os.RemoveAll(kms.saTokenDir)
}
@ -158,7 +158,7 @@ func (kms *VaultTenantSA) Destroy() {
kms.vaultTenantConnection.Destroy()
}
func (kms *VaultTenantSA) configureTenant(config map[string]interface{}, tenant string) error {
func (kms *vaultTenantSA) configureTenant(config map[string]interface{}, tenant string) error {
kms.Tenant = tenant
tenantConfig, found := fetchTenantConfig(config, tenant)
if found {
@ -184,11 +184,11 @@ func (kms *VaultTenantSA) configureTenant(config map[string]interface{}, tenant
}
// parseConfig calls vaultTenantConnection.parseConfig() and also set
// additional config options specific to VaultTenantSA. This function is called
// additional config options specific to vaultTenantSA. This function is called
// multiple times, for the different nested configuration layers.
// parseTenantConfig() calls this as well, with a reduced set of options,
// filtered by isTenantConfigOption().
func (kms *VaultTenantSA) parseConfig(config map[string]interface{}) error {
func (kms *vaultTenantSA) parseConfig(config map[string]interface{}) error {
err := kms.vaultTenantConnection.parseConfig(config)
if err != nil {
return err
@ -234,7 +234,7 @@ func isTenantSAConfigOption(opt string) bool {
return true
}
// additional options for VaultTenantSA
// additional options for vaultTenantSA
switch opt {
case "tenantSAName":
case "vaultAuthPath":
@ -248,7 +248,7 @@ func isTenantSAConfigOption(opt string) bool {
// setServiceAccountName stores the name of the ServiceAccount in the
// configuration if it has been set in the options.
func (kms *VaultTenantSA) setServiceAccountName(config map[string]interface{}) error {
func (kms *vaultTenantSA) setServiceAccountName(config map[string]interface{}) error {
err := setConfigString(&kms.tenantSAName, config, "tenantSAName")
if errors.Is(err, errConfigOptionInvalid) {
return err
@ -258,8 +258,8 @@ func (kms *VaultTenantSA) setServiceAccountName(config map[string]interface{}) e
}
// getServiceAccount returns the Tenants ServiceAccount with the name
// configured in the VaultTenantSA.
func (kms *VaultTenantSA) getServiceAccount() (*corev1.ServiceAccount, error) {
// configured in the vaultTenantSA.
func (kms *vaultTenantSA) getServiceAccount() (*corev1.ServiceAccount, error) {
c, err := kms.getK8sClient()
if err != nil {
return nil, fmt.Errorf("can not get ServiceAccount %s/%s, "+
@ -278,7 +278,7 @@ func (kms *VaultTenantSA) getServiceAccount() (*corev1.ServiceAccount, error) {
// getToken looks up the ServiceAccount and the Secrets linked from it. When it
// finds the Secret that contains the `token` field, the contents is read and
// returned.
func (kms *VaultTenantSA) getToken() (string, error) {
func (kms *vaultTenantSA) getToken() (string, error) {
sa, err := kms.getServiceAccount()
if err != nil {
return "", err
@ -309,7 +309,7 @@ func (kms *VaultTenantSA) getToken() (string, error) {
// getTokenPath creates a temporary directory structure that contains the token
// linked from the ServiceAccount. This path can then be used in place of the
// standard `/var/run/secrets/kubernetes.io/serviceaccount/token` location.
func (kms *VaultTenantSA) getTokenPath() (string, error) {
func (kms *vaultTenantSA) getTokenPath() (string, error) {
dir, err := ioutil.TempDir("", kms.tenantSAName)
if err != nil {
return "", fmt.Errorf("failed to create directory for ServiceAccount %s/%s: %w", kms.tenantSAName, kms.Tenant, err)

2
internal/kms/vault_sa_test.go
View File

@ -31,7 +31,7 @@ func TestVaultTenantSAKMSRegistered(t *testing.T) {
func TestTenantSAParseConfig(t *testing.T) {
t.Parallel()
vts := VaultTenantSA{}
vts := vaultTenantSA{}
config := make(map[string]interface{})

12
internal/kms/vault_tokens.go
View File

@ -186,7 +186,7 @@ Example JSON structure in the KMS config is,
*/
type vaultTenantConnection struct {
vaultConnection
IntegratedDEK
integratedDEK
client *kubernetes.Clientset
@ -202,7 +202,7 @@ type vaultTenantConnection struct {
tenantConfigOptionFilter func(string) bool
}
type VaultTokensKMS struct {
type vaultTokensKMS struct {
vaultTenantConnection
// TokenName is the name of the Secret in the Tenants Kubernetes Namespace
@ -228,7 +228,7 @@ func initVaultTokensKMS(args ProviderInitArgs) (EncryptionKMS, error) {
}
}
kms := &VaultTokensKMS{}
kms := &vaultTokensKMS{}
kms.vaultTenantConnection.init()
err = kms.initConnection(config)
if err != nil {
@ -278,7 +278,7 @@ func initVaultTokensKMS(args ProviderInitArgs) (EncryptionKMS, error) {
return kms, nil
}
func (kms *VaultTokensKMS) configureTenant(config map[string]interface{}, tenant string) error {
func (kms *vaultTokensKMS) configureTenant(config map[string]interface{}, tenant string) error {
kms.Tenant = tenant
tenantConfig, found := fetchTenantConfig(config, tenant)
if found {
@ -340,7 +340,7 @@ func (vtc *vaultTenantConnection) parseConfig(config map[string]interface{}) err
// setTokenName updates the kms.TokenName with the options from config. This
// method can be called multiple times, i.e. to override configuration options
// from tenants.
func (kms *VaultTokensKMS) setTokenName(config map[string]interface{}) error {
func (kms *vaultTokensKMS) setTokenName(config map[string]interface{}) error {
err := setConfigString(&kms.TokenName, config, "tenantTokenName")
if errors.Is(err, errConfigOptionInvalid) {
return err
@ -501,7 +501,7 @@ func (vtc *vaultTenantConnection) RemoveDEK(key string) error {
return nil
}
func (kms *VaultTokensKMS) getToken() (string, error) {
func (kms *vaultTokensKMS) getToken() (string, error) {
c, err := kms.getK8sClient()
if err != nil {
return "", err

9
internal/kms/vault_tokens_test.go
View File

@ -23,6 +23,8 @@ import (
"strings"
"testing"
"github.com/hashicorp/vault/api"
loss "github.com/libopenstorage/secrets"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -41,6 +43,8 @@ func TestParseConfig(t *testing.T) {
// fill default options (normally done in initVaultTokensKMS)
config["vaultAddress"] = "https://vault.default.cluster.svc"
config["vaultNamespace"] = "default"
config["vaultAuthNamespace"] = "company-sso"
config["tenantConfigName"] = vaultTokensDefaultConfigName
// parsing with all required options
@ -55,12 +59,17 @@ func TestParseConfig(t *testing.T) {
// tenant "bob" uses a different kms.ConfigName
bob := make(map[string]interface{})
bob["tenantConfigName"] = "the-config-from-bob"
bob["vaultNamespace"] = "bobs-place"
err = vtc.parseConfig(bob)
switch {
case err != nil:
t.Errorf("unexpected error: %s", err)
case vtc.ConfigName != "the-config-from-bob":
t.Errorf("ConfigName contains unexpected value: %s", vtc.ConfigName)
case vtc.vaultConfig[api.EnvVaultNamespace] != "company-sso":
t.Errorf("EnvVaultNamespace contains unexpected value: %s", vtc.vaultConfig[api.EnvVaultNamespace])
case vtc.keyContext[loss.KeyVaultNamespace] != "bobs-place":
t.Errorf("KeyVaultNamespace contains unexpected value: %s", vtc.keyContext[loss.KeyVaultNamespace])
}
}

45
internal/rbd/clone.go
View File

@ -167,13 +167,6 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol
}
}
if rv.ThickProvision {
err = rv.setThickProvisioned()
if err != nil {
return fmt.Errorf("failed mark %q thick-provisioned: %w", rv, err)
}
}
err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID)
if err != nil {
log.ErrorLog(ctx, "failed to store volume %s: %v", rv, err)
@ -236,39 +229,27 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro
}
}()
if rv.ThickProvision {
err = tempClone.DeepCopy(&rv.rbdImage)
if err != nil {
return fmt.Errorf("failed to deep copy %q into %q: %w", parentVol, rv, err)
}
} else {
// flatten clone
errFlatten = tempClone.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if errFlatten != nil {
return errFlatten
}
// flatten clone
errFlatten = tempClone.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if errFlatten != nil {
return errFlatten
}
// create snap of temp clone from temporary cloned image
// create final clone
// delete snap of temp clone
errClone = createRBDClone(ctx, tempClone, rv, cloneSnap)
if errClone != nil {
// set errFlatten error to cleanup temporary snapshot and temporary clone
errFlatten = errors.New("failed to create user requested cloned image")
// create snap of temp clone from temporary cloned image
// create final clone
// delete snap of temp clone
errClone = createRBDClone(ctx, tempClone, rv, cloneSnap)
if errClone != nil {
// set errFlatten error to cleanup temporary snapshot and temporary clone
errFlatten = errors.New("failed to create user requested cloned image")
return errClone
}
return errClone
}
return nil
}
func (rv *rbdVolume) flattenCloneImage(ctx context.Context) error {
if rv.ThickProvision {
// thick-provisioned images do not need flattening
return nil
}
tempClone := rv.generateTempClone()
// reducing the limit for cloned images to make sure the limit is in range,
// If the intermediate clone reaches the depth we may need to return ABORT

203
internal/rbd/controllerserver.go
View File

@ -19,8 +19,6 @@ package rbd
import (
"context"
"errors"
"fmt"
"strconv"
csicommon "github.com/ceph/ceph-csi/internal/csi-common"
"github.com/ceph/ceph-csi/internal/util"
@ -132,8 +130,6 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error())
}
rbdVol.ThickProvision = isThickProvisionRequest(req.GetParameters())
rbdVol.RequestName = req.GetName()
// Volume Size - Default is 1 GiB
@ -211,11 +207,6 @@ func checkValidCreateVolumeRequest(rbdVol, parentVol *rbdVolume, rbdSnap *rbdSna
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
}
err = rbdSnap.isCompatibleThickProvision(rbdVol)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
}
err = rbdSnap.isCompabitableClone(&rbdVol.rbdImage)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
@ -227,11 +218,6 @@ func checkValidCreateVolumeRequest(rbdVol, parentVol *rbdVolume, rbdSnap *rbdSna
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
}
err = parentVol.isCompatibleThickProvision(rbdVol)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
}
err = parentVol.isCompabitableClone(&rbdVol.rbdImage)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
@ -287,7 +273,7 @@ func (cs *ControllerServer) CreateVolume(
if err != nil {
return nil, getGRPCErrorForCreateVolume(err)
} else if found {
return cs.repairExistingVolume(ctx, req, cr, rbdVol, parentVol, rbdSnap)
return cs.repairExistingVolume(ctx, req, cr, rbdVol, rbdSnap)
}
err = checkValidCreateVolumeRequest(rbdVol, parentVol, rbdSnap)
@ -352,50 +338,12 @@ func flattenParentImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent
// that the state is corrected to what was requested. It is needed to call this
// when the process of creating a volume was interrupted.
func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.CreateVolumeRequest,
cr *util.Credentials, rbdVol, parentVol *rbdVolume, rbdSnap *rbdSnapshot) (*csi.CreateVolumeResponse, error) {
cr *util.Credentials, rbdVol *rbdVolume, rbdSnap *rbdSnapshot) (*csi.CreateVolumeResponse, error) {
vcs := req.GetVolumeContentSource()
switch {
// normal CreateVolume without VolumeContentSource
case vcs == nil:
// continue/restart allocating the volume in case it
// should be thick-provisioned
if isThickProvisionRequest(req.GetParameters()) {
err := rbdVol.RepairThickProvision()
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}
// rbdVol is a restore from snapshot, rbdSnap is passed
case vcs.GetSnapshot() != nil:
// When restoring of a thick-provisioned volume was happening,
// the image should be marked as thick-provisioned, unless it
// was aborted in flight. In order to restart the
// thick-restoring, delete the volume and let the caller retry
// from the start.
if isThickProvisionRequest(req.GetParameters()) {
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(
codes.Aborted,
"failed to verify thick-provisioned volume %q: %s",
rbdVol,
err)
} else if !thick {
err = rbdVol.deleteImage(ctx)
if err != nil {
return nil, status.Errorf(codes.Aborted, "failed to remove partially cloned volume %q: %s", rbdVol, err)
}
err = undoVolReservation(ctx, rbdVol, cr)
if err != nil {
return nil, status.Errorf(codes.Aborted, "failed to remove volume %q from journal: %s", rbdVol, err)
}
return nil, status.Errorf(
codes.Aborted,
"restoring thick-provisioned volume %q has been interrupted, please retry", rbdVol)
}
}
// restore from snapshot implies rbdSnap != nil
// check if image depth is reached limit and requires flatten
err := checkFlatten(ctx, rbdVol, cr)
@ -418,23 +366,6 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
// rbdVol is a clone from parentVol
case vcs.GetVolume() != nil:
// When cloning into a thick-provisioned volume was happening,
// the image should be marked as thick-provisioned, unless it
// was aborted in flight. In order to restart the
// thick-cloning, delete the volume and undo the reservation in
// the journal to let the caller retry from the start.
if isThickProvisionRequest(req.GetParameters()) {
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(
codes.Internal,
"failed to verify thick-provisioned volume %q: %s",
rbdVol,
err)
} else if !thick {
return nil, cleanupThickClone(ctx, parentVol, rbdVol, rbdSnap, cr)
}
}
// expand the image if the requested size is greater than the current size
err := rbdVol.expand()
if err != nil {
@ -447,26 +378,6 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
return buildCreateVolumeResponse(req, rbdVol), nil
}
// cleanupThickClone will delete the snapshot and volume and undo the reservation.
func cleanupThickClone(ctx context.Context,
rbdVol,
parentVol *rbdVolume,
rbdSnap *rbdSnapshot,
cr *util.Credentials) error {
err := cleanUpSnapshot(ctx, parentVol, rbdSnap, rbdVol)
if err != nil {
return status.Errorf(codes.Internal, "failed to remove partially cloned volume %q: %s", rbdVol, err)
}
err = undoVolReservation(ctx, rbdVol, cr)
if err != nil {
return status.Errorf(codes.Internal, "failed to remove volume %q from journal: %s", rbdVol, err)
}
return status.Errorf(
codes.Internal,
"cloning thick-provisioned volume %q has been interrupted, please retry", rbdVol)
}
// check snapshots on the rbd image, as we have limit from krbd that an image
// cannot have more than 510 snapshot at a given point of time. If the
// snapshots are more than the `maxSnapshotsOnImage` Add a task to flatten all
@ -474,11 +385,6 @@ func cleanupThickClone(ctx context.Context,
// are more than the `minSnapshotOnImage` Add a task to flatten all the
// temporary cloned images.
func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
if rbdVol.ThickProvision {
// thick-provisioned images do not need flattening
return nil
}
snaps, err := rbdVol.listSnapshots()
if err != nil {
if errors.Is(err, ErrImageNotFound) {
@ -592,27 +498,12 @@ func (cs *ControllerServer) createVolumeFromSnapshot(
// as we are operating on single cluster reuse the connection
parentVol.conn = rbdVol.conn.Copy()
if rbdVol.ThickProvision {
err = parentVol.DeepCopy(&rbdVol.rbdImage)
if err != nil {
return status.Errorf(codes.Internal, "failed to deep copy %q into %q: %v", parentVol, rbdVol, err)
}
err = rbdVol.setThickProvisioned()
if err != nil {
return status.Errorf(codes.Internal, "failed to mark %q thick-provisioned: %s", rbdVol, err)
}
err = parentVol.copyEncryptionConfig(&rbdVol.rbdImage, true)
if err != nil {
return status.Errorf(codes.Internal, err.Error())
}
} else {
// create clone image and delete snapshot
err = rbdVol.cloneRbdImageFromSnapshot(ctx, rbdSnap, parentVol)
if err != nil {
log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err)
// create clone image and delete snapshot
err = rbdVol.cloneRbdImageFromSnapshot(ctx, rbdSnap, parentVol)
if err != nil {
log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err)
return err
}
return err
}
log.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol, rbdSnap)
@ -1136,31 +1027,6 @@ func cloneFromSnapshot(
}
}
// The clone image created during CreateSnapshot has to be marked as thick.
// As snapshot and volume both are independent we cannot depend on the
// parent volume of the clone to check thick provision during CreateVolume
// from snapshot operation because the parent volume can be deleted anytime
// after snapshot is created.
// TODO: copy thick provision config
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed checking thick-provisioning of %q: %s", rbdVol, err)
}
if thick {
// check the thick metadata is already set on the clone image.
thick, err = vol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed checking thick-provisioning of %q: %s", vol, err)
}
if !thick {
err = vol.setThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed mark %q thick-provisioned: %s", vol, err)
}
}
}
err = vol.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if errors.Is(err, ErrFlattenInProgress) {
// if flattening is in progress, return error and do not cleanup
@ -1258,30 +1124,13 @@ func (cs *ControllerServer) doSnapshotClone(
}
}
// The clone image created during CreateSnapshot has to be marked as thick.
// As snapshot and volume both are independent we cannot depend on the
// parent volume of the clone to check thick provision during CreateVolume
// from snapshot operation because the parent volume can be deleted anytime
// after snapshot is created.
thick, err := parentVol.isThickProvisioned()
err = cloneRbd.createSnapshot(ctx, rbdSnap)
if err != nil {
return nil, fmt.Errorf("failed checking thick-provisioning of %q: %w", parentVol, err)
}
// update rbd image name for logging
rbdSnap.RbdImageName = cloneRbd.RbdImageName
log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err)
if thick {
err = cloneRbd.setThickProvisioned()
if err != nil {
return nil, fmt.Errorf("failed mark %q thick-provisioned: %w", cloneRbd, err)
}
} else {
err = cloneRbd.createSnapshot(ctx, rbdSnap)
if err != nil {
// update rbd image name for logging
rbdSnap.RbdImageName = cloneRbd.RbdImageName
log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err)
return cloneRbd, err
}
return cloneRbd, err
}
err = cloneRbd.getImageID()
@ -1542,31 +1391,3 @@ func (cs *ControllerServer) ControllerExpandVolume(
NodeExpansionRequired: nodeExpansion,
}, nil
}
// logThickProvisioningDeprecation makes sure the deprecation warning about
// thick-provisining is logged only once.
var logThickProvisioningDeprecation = true
// isThickProvisionRequest returns true in case the request contains the
// `thickProvision` option set to `true`.
func isThickProvisionRequest(parameters map[string]string) bool {
tp := "thickProvision"
thick, ok := parameters[tp]
if !ok || thick == "" {
return false
}
thickBool, err := strconv.ParseBool(thick)
if err != nil {
return false
}
if logThickProvisioningDeprecation {
log.WarningLogMsg("thick-provisioning is deprecated and will " +
"be removed in a future release")
logThickProvisioningDeprecation = false
}
return thickBool
}

69
internal/rbd/controllerserver_test.go
View File

@ -1,69 +0,0 @@
/*
Copyright 2021 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
)
func TestIsThickProvisionRequest(t *testing.T) {
t.Parallel()
req := &csi.CreateVolumeRequest{
Name: "fake",
Parameters: map[string]string{
"unkownOption": "not-set",
},
}
// pass disabled/invalid values for "thickProvision" option
if isThickProvisionRequest(req.GetParameters()) {
t.Error("request is not for thick-provisioning")
}
req.Parameters["thickProvision"] = ""
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "false"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "off"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "no"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "**true**"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
// only "true" should enable thick provisioning
req.Parameters["thickProvision"] = "true"
if !isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request should be for thick-provisioning: %s", req.Parameters["thickProvision"])
}
}

1
internal/rbd/nodeserver.go
View File

@ -178,7 +178,6 @@ func populateRbdVol(
return nil, status.Error(codes.Internal, err.Error())
}
rv.ThickProvision = isThickProvisionRequest(req.GetVolumeContext())
isStaticVol := parseBoolOption(ctx, req.GetVolumeContext(), staticVol, false)
// get rbd image name from the volume journal
// for static volumes, the image name is actually the volume ID itself

37
internal/rbd/rbd_attach.go
View File

@ -310,7 +310,7 @@ func attachRBDImage(ctx context.Context, volOptions *rbdVolume, device string, c
return devicePath, err
}
func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions, cookie string) []string {
func appendNbdDeviceTypeAndOptions(cmdArgs []string, userOptions, cookie string) []string {
cmdArgs = append(cmdArgs, "--device-type", accessTypeNbd)
isUnmap := CheckSliceContains(cmdArgs, "unmap")
@ -328,12 +328,6 @@ func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions,
if hasNBDCookieSupport {
cmdArgs = append(cmdArgs, "--options", fmt.Sprintf("cookie=%s", cookie))
}
if isThick {
// When an image is thick-provisioned, any discard/unmap/trim
// requests should not free extents.
cmdArgs = append(cmdArgs, "--options", "notrim")
}
}
if userOptions != "" {
@ -345,22 +339,11 @@ func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions,
return cmdArgs
}
func appendKRbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions string) []string {
cmdArgs = append(cmdArgs, "--device-type", accessTypeKRbd)
isUnmap := CheckSliceContains(cmdArgs, "unmap")
if !isUnmap {
if isThick {
// When an image is thick-provisioned, any discard/unmap/trim
// requests should not free extents.
cmdArgs = append(cmdArgs, "--options", "notrim")
}
}
func appendKRbdDeviceTypeAndOptions(cmdArgs []string, userOptions string) []string {
// Enable mapping and unmapping images from a non-initial network
// namespace (e.g. for Multus CNI). The network namespace must be
// owned by the initial user namespace.
cmdArgs = append(cmdArgs, "--options", "noudev")
cmdArgs = append(cmdArgs, "--device-type", accessTypeKRbd, "--options", "noudev")
if userOptions != "" {
// userOptions is appended after, possibly overriding the above
@ -413,12 +396,6 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util.
isNbd = true
}
// check if the image should stay thick-provisioned
isThick, err := volOpt.isThickProvisioned()
if err != nil {
log.WarningLog(ctx, "failed to detect if image %q is thick-provisioned: %v", volOpt, err)
}
if isNbd {
mapArgs = append(mapArgs, "--log-file",
getCephClientLogFileName(volOpt.VolID, volOpt.LogDir, "rbd-nbd"))
@ -433,9 +410,9 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util.
} else {
mapArgs = append(mapArgs, "map", imagePath)
if isNbd {
mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions, volOpt.VolID)
mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, volOpt.MapOptions, volOpt.VolID)
} else {
mapArgs = appendKRbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions)
mapArgs = appendKRbdDeviceTypeAndOptions(mapArgs, volOpt.MapOptions)
}
}
@ -543,9 +520,9 @@ func detachRBDImageOrDeviceSpec(
unmapArgs := []string{"unmap", dArgs.imageOrDeviceSpec}
if dArgs.isNbd {
unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions, dArgs.volumeID)
unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, dArgs.unmapOptions, dArgs.volumeID)
} else {
unmapArgs = appendKRbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions)
unmapArgs = appendKRbdDeviceTypeAndOptions(unmapArgs, dArgs.unmapOptions)
}
_, stderr, err := util.ExecCommand(ctx, rbd, unmapArgs...)

248
internal/rbd/rbd_util.go
View File

@ -58,18 +58,6 @@ const (
rbdTaskRemoveCmdInvalidString = "No handler found"
rbdTaskRemoveCmdAccessDeniedMessage = "access denied:"
// image metadata key for thick-provisioning.
// As image metadata key starting with '.rbd' will not be copied when we do
// clone or mirroring, deprecating the old key for the same reason use
// 'thickProvisionMetaKey' to set image metadata.
deprecatedthickProvisionMetaKey = ".rbd.csi.ceph.com/thick-provisioned"
thickProvisionMetaKey = "rbd.csi.ceph.com/thick-provisioned"
// these are the metadata set on the image to identify the image is
// thick provisioned or thin provisioned.
thickProvisionMetaData = "true"
thinProvisionMetaData = "false"
// migration label key and value for parameters in volume context.
intreeMigrationKey = "migration"
intreeMigrationLabel = "true"
@ -173,7 +161,6 @@ type rbdVolume struct {
RequestedVolSize int64
DisableInUseChecks bool
readOnly bool
ThickProvision bool
}
// rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics.
@ -382,26 +369,6 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er
}
}
if pOpts.ThickProvision {
err = pOpts.allocate(0)
if err != nil {
// nolint:errcheck // deleteImage() will log errors in
// case it fails, no need to log them here again
_ = pOpts.deleteImage(ctx)
return fmt.Errorf("failed to thick provision image: %w", err)
}
err = pOpts.setThickProvisioned()
if err != nil {
// nolint:errcheck // deleteImage() will log errors in
// case it fails, no need to log them here again
_ = pOpts.deleteImage(ctx)
return fmt.Errorf("failed to mark image as thick-provisioned: %w", err)
}
}
return nil
}
@ -465,82 +432,6 @@ func (ri *rbdImage) open() (*librbd.Image, error) {
return image, nil
}
// allocate uses the stripe-period of the image to fully allocate (thick
// provision) the image.
func (ri *rbdImage) allocate(offset uint64) error {
// We do not want to call discard, we really want to write zeros to get
// the allocation. This sets the option for the re-used connection, and
// all subsequent images that are opened. That is not a problem, as
// this is the only place images get written.
err := ri.conn.DisableDiscardOnZeroedWriteSame()
if err != nil {
return err
}
image, err := ri.open()
if err != nil {
return err
}
defer image.Close()
st, err := image.Stat()
if err != nil {
return err
}
sc, err := image.GetStripeCount()
if err != nil {
return err
}
// blockSize is the stripe-period: size of the object-size multiplied
// by the stripe-count
blockSize := sc * (1 << st.Order)
zeroBlock := make([]byte, blockSize)
// the actual size of the image as available in the pool, can be
// marginally different from the requested image size
size := st.Size - offset
// In case the remaining space on the volume is smaller than blockSize,
// write a partial block with WriteAt() after this loop.
for size > blockSize {
writeSize := size
// write a maximum of 1GB per WriteSame() call
if size > helpers.GiB {
writeSize = helpers.GiB
}
// round down to the size of a zeroBlock
if (writeSize % blockSize) != 0 {
writeSize = (writeSize / blockSize) * blockSize
}
_, err = image.WriteSame(offset, writeSize, zeroBlock,
rados.OpFlagNone)
if err != nil {
return fmt.Errorf("failed to allocate %d/%d bytes at "+
"offset %d: %w", writeSize, blockSize, offset, err)
}
// write succeeded
size -= writeSize
offset += writeSize
}
// write the last remaining bytes, in case the image size can not be
// written with the optimal blockSize
if size != 0 {
_, err = image.WriteAt(zeroBlock[:size], int64(offset))
if err != nil {
return fmt.Errorf("failed to allocate %d bytes at "+
"offset %d: %w", size, offset, err)
}
}
return nil
}
// isInUse checks if there is a watcher on the image. It returns true if there
// is a watcher on the image, otherwise returns false.
func (ri *rbdImage) isInUse() (bool, error) {
@ -1710,40 +1601,10 @@ func (ri *rbdImage) resize(newSize int64) error {
}
defer image.Close()
thick, err := ri.isThickProvisioned()
if err != nil {
return err
}
// offset is used to track from where on the expansion is done, so that
// the extents can be allocated in case the image is thick-provisioned
var offset uint64
if thick {
st, statErr := image.Stat()
if statErr != nil {
return statErr
}
offset = st.Size
}
err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB))
if err != nil {
return err
}
if thick {
err = ri.allocate(offset)
if err != nil {
resizeErr := image.Resize(offset)
if resizeErr != nil {
err = fmt.Errorf("failed to shrink image (%v) after failed allocation: %w", resizeErr, err)
}
return err
}
}
// update Volsize of rbdVolume object to newSize.
ri.VolSize = newSize
@ -1820,71 +1681,6 @@ func (ri *rbdImage) MigrateMetadata(oldKey, newKey, defaultValue string) (string
return value, nil
}
// setThickProvisioned records in the image metadata that it has been
// thick-provisioned.
func (ri *rbdImage) setThickProvisioned() error {
err := ri.SetMetadata(thickProvisionMetaKey, thickProvisionMetaData)
if err != nil {
return fmt.Errorf("failed to set metadata %q for %q: %w", thickProvisionMetaKey, ri, err)
}
return nil
}
// isThickProvisioned checks in the image metadata if the image has been marked
// as thick-provisioned. This can be used while expanding the image, so that
// the expansion can be allocated too.
func (ri *rbdImage) isThickProvisioned() (bool, error) {
value, err := ri.MigrateMetadata(deprecatedthickProvisionMetaKey, thickProvisionMetaKey, thinProvisionMetaData)
if err != nil {
return false, fmt.Errorf("failed to get metadata %q for %q: %w", thickProvisionMetaKey, ri, err)
}
thick, err := strconv.ParseBool(value)
if err != nil {
return false, fmt.Errorf("failed to convert %q=%q to a boolean: %w", thickProvisionMetaKey, value, err)
}
return thick, nil
}
// RepairThickProvision writes zero bytes to the volume so that it will be
// completely allocated. In case the volume is already marked as
// thick-provisioned, nothing will be done.
func (ri *rbdImage) RepairThickProvision() error {
// if the image has the thick-provisioned metadata, it has been fully
// allocated
done, err := ri.isThickProvisioned()
if err != nil {
return fmt.Errorf("failed to repair thick-provisioning of %q: %w", ri, err)
} else if done {
return nil
}
// in case there are watchers, assume allocating is still happening in
// the background (by an other process?)
background, err := ri.isInUse()
if err != nil {
return fmt.Errorf("failed to get users of %q: %w", ri, err)
} else if background {
return fmt.Errorf("not going to restart thick-provisioning of in-use image %q", ri)
}
// TODO: can this be improved by starting at the offset where
// allocating was aborted/restarted?
err = ri.allocate(0)
if err != nil {
return fmt.Errorf("failed to continue thick-provisioning of %q: %w", ri, err)
}
err = ri.setThickProvisioned()
if err != nil {
return fmt.Errorf("failed to continue thick-provisioning of %q: %w", ri, err)
}
return nil
}
// DeepCopy creates an independent image (dest) from the source image. This
// process may take some time when the image is large.
func (ri *rbdImage) DeepCopy(dest *rbdImage) error {
@ -2004,50 +1800,6 @@ func (ri *rbdImage) isCompabitableClone(dst *rbdImage) error {
return nil
}
func (ri *rbdImage) isCompatibleThickProvision(dst *rbdVolume) error {
thick, err := ri.isThickProvisioned()
if err != nil {
return err
}
switch {
case thick && !dst.ThickProvision:
return fmt.Errorf("cannot create thin volume from thick volume %q", ri)
case !thick && dst.ThickProvision:
return fmt.Errorf("cannot create thick volume from thin volume %q", ri)
}
return nil
}
// FIXME: merge isCompatibleThickProvision of rbdSnapshot and rbdImage to a single
// function.
func (rs *rbdSnapshot) isCompatibleThickProvision(dst *rbdVolume) error {
// During CreateSnapshot the rbd image will be created with the
// snapshot name. Replacing RbdImageName with RbdSnapName so that we
// can check if the image is thick provisioned
vol := generateVolFromSnap(rs)
err := vol.Connect(rs.conn.Creds)
if err != nil {
return err
}
defer vol.Destroy()
thick, err := vol.isThickProvisioned()
if err != nil {
return err
}
switch {
case thick && !dst.ThickProvision:
return fmt.Errorf("cannot create thin volume from thick volume %q", vol)
case !thick && dst.ThickProvision:
return fmt.Errorf("cannot create thick volume from thin volume %q", vol)
}
return nil
}
func (ri *rbdImage) addSnapshotScheduling(
interval admin.Interval,
startTime admin.StartTime) error {

8
internal/util/cephconf.go
View File

@ -24,14 +24,6 @@ var cephConfig = []byte(`[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
# ceph-fuse which uses libfuse2 by default has write buffer size of 2KiB
# adding 'fuse_big_writes = true' option by default to override this limit
# see https://github.com/ceph/ceph-csi/issues/1928
fuse_big_writes = true
`)
const (

20
internal/util/connection.go
View File

@ -140,23 +140,3 @@ func (cc *ClusterConnection) GetTaskAdmin() (*ra.TaskAdmin, error) {
return rbdAdmin.Task(), nil
}
// DisableDiscardOnZeroedWriteSame enables the
// `rbd_discard_on_zeroed_write_same` option in the cluster connection, so that
// writing zero blocks of data are actual writes on the OSDs (doing
// allocations) and not discard calls. This makes writes much slower, but
// enables the option to do thick-provisioning.
func (cc *ClusterConnection) DisableDiscardOnZeroedWriteSame() error {
if cc.discardOnZeroedWriteSameDisabled {
return nil
}
err := cc.conn.SetConfigOption("rbd_discard_on_zeroed_write_same", "false")
if err != nil {
return err
}
cc.discardOnZeroedWriteSameDisabled = true
return nil
}

5
scripts/Dockerfile.devel
View File

@ -17,6 +17,11 @@ RUN source /build.env \
&& mkdir -p /usr/local/go \
&& curl https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${GOARCH}.tar.gz | tar xzf - -C ${GOROOT} --strip-components=1
# FIXME: Ceph does not need Apache Arrow anymore, some container images may
# still have the repository enabled. Disabling the repository can be removed in
# the future, see https://github.com/ceph/ceph-container/pull/1990 .
RUN dnf config-manager --disable apache-arrow-centos || true
RUN dnf -y install \
git \
make \

11
scripts/install-snapshot.sh
View File

@ -7,7 +7,7 @@ SCRIPT_DIR="$(dirname "${0}")"
# shellcheck source=build.env
source "${SCRIPT_DIR}/../build.env"
SNAPSHOT_VERSION=${SNAPSHOT_VERSION:-"v4.0.0"}
SNAPSHOT_VERSION=${SNAPSHOT_VERSION:-"v5.0.1"}
TEMP_DIR="$(mktemp -d)"
SNAPSHOTTER_URL="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${SNAPSHOT_VERSION}"
@ -24,7 +24,7 @@ VOLUME_SNAPSHOT="${SNAPSHOTTER_URL}/client/config/crd/snapshot.storage.k8s.io_vo
function install_snapshot_controller() {
local namespace=$1
if [ -z "${namespace}" ]; then
namespace="default"
namespace="kube-system"
fi
create_or_delete_resource "create" ${namespace}
@ -51,7 +51,7 @@ function install_snapshot_controller() {
function cleanup_snapshot_controller() {
local namespace=$1
if [ -z "${namespace}" ]; then
namespace="default"
namespace="kube-system"
fi
create_or_delete_resource "delete" ${namespace}
}
@ -65,8 +65,9 @@ function create_or_delete_resource() {
mkdir -p "${TEMP_DIR}"
curl -o "${temp_rbac}" "${SNAPSHOT_RBAC}"
curl -o "${temp_snap_controller}" "${SNAPSHOT_CONTROLLER}"
sed -i "s/namespace: default/namespace: ${namespace}/g" "${temp_rbac}"
sed -i "s/namespace: default/namespace: ${namespace}/g" "${snapshotter_psp}"
sed -i "s/namespace: kube-system/namespace: ${namespace}/g" "${temp_rbac}"
sed -i "s/namespace: kube-system/namespace: ${namespace}/g" "${temp_snap_controller}"
sed -i "s/namespace: kube-system/namespace: ${namespace}/g" "${snapshotter_psp}"
sed -i "s/canary/${SNAPSHOT_VERSION}/g" "${temp_snap_controller}"
kubectl "${operation}" -f "${temp_rbac}"

13
scripts/snapshot-controller-psp.yaml
View File

@ -3,6 +3,7 @@ apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: csi-snapshotter-psp
namespace: kube-system
spec:
allowPrivilegeEscalation: true
allowedCapabilities:
@ -28,8 +29,8 @@ kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-snapshotter-psp
# replace with non-default namespace name
namespace: default
# replace with non-kube-system namespace name
namespace: kube-system
rules:
- apiGroups: ["policy"]
resources: ["podsecuritypolicies"]
@ -41,13 +42,13 @@ kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-snapshotter-psp
# replace with non-default namespace name
namespace: default
# replace with non-kube-system namespace name
namespace: kube-system
subjects:
- kind: ServiceAccount
name: snapshot-controller
# replace with non-default namespace name
namespace: default
# replace with non-kube-system namespace name
namespace: kube-system
roleRef:
kind: Role
name: csi-snapshotter-psp

18
vendor/github.com/onsi/gomega/.travis.yml generated vendored
View File

@ -1,18 +0,0 @@
language: go
arch:
- amd64
- ppc64le
go:
- gotip
- 1.16.x
- 1.15.x
env:
- GO111MODULE=on
install: skip
script:
- go mod tidy && git diff --exit-code go.mod go.sum
- make test

17
vendor/github.com/onsi/gomega/CHANGELOG.md generated vendored
View File

@ -1,3 +1,20 @@
## 1.18.0
## Features
- Docs now live on the master branch in the docs folder which will make for easier PRs. The docs also use Ginkgo 2.0's new docs html/css/js. [2570272]
- New HaveValue matcher can handle actuals that are either values (in which case they are passed on unscathed) or pointers (in which case they are indirected). [Docs here.](https://onsi.github.io/gomega/#working-with-values) (#485) [bdc087c]
- Gmeasure has been declared GA [360db9d]
## Fixes
- Gomega now uses ioutil for Go 1.15 and lower (#492) - official support is only for the most recent two major versions of Go but this will unblock users who need to stay on older unsupported versions of Go. [c29c1c0]
## Maintenace
- Remove Travis workflow (#491) [72e6040]
- Upgrade to Ginkgo 2.0.0 GA [f383637]
- chore: fix description of HaveField matcher (#487) [2b4b2c0]
- use tools.go to ensure Ginkgo cli dependencies are included [f58a52b]
- remove dockerfile and simplify github actions to match ginkgo's actions [3f8160d]
## 1.17.0
### Features

1
vendor/github.com/onsi/gomega/Dockerfile generated vendored
View File

@ -1 +0,0 @@
FROM golang:1.15

33
vendor/github.com/onsi/gomega/Makefile generated vendored
View File

@ -1,33 +0,0 @@
###### Help ###################################################################
.DEFAULT_GOAL = help
.PHONY: help
help: ## list Makefile targets
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
###### Targets ################################################################
test: version download fmt vet ginkgo ## Runs all build, static analysis, and test steps
download: ## Download dependencies
go mod download
vet: ## Run static code analysis
go vet ./...
ginkgo: ## Run tests using Ginkgo
go run github.com/onsi/ginkgo/ginkgo -p -r --randomizeAllSpecs --failOnPending --randomizeSuites --race
fmt: ## Checks that the code is formatted correcty
@@if [ -n "$$(gofmt -s -e -l -d .)" ]; then \
echo "gofmt check failed: run 'gofmt -s -e -l -w .'"; \
exit 1; \
fi
docker_test: ## Run tests in a container via docker-compose
docker-compose build test && docker-compose run --rm test make test
version: ## Display the version of Go
@@go version

10
vendor/github.com/onsi/gomega/docker-compose.yaml generated vendored
View File

@ -1,10 +0,0 @@
version: '3.0'
services:
test:
build:
dockerfile: Dockerfile
context: .
working_dir: /app
volumes:
- ${PWD}:/app

2
vendor/github.com/onsi/gomega/gomega_dsl.go generated vendored
View File

@ -22,7 +22,7 @@ import (
"github.com/onsi/gomega/types"
)
const GOMEGA_VERSION = "1.17.0"
const GOMEGA_VERSION = "1.18.0"
const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler.
If you're using Ginkgo then you probably forgot to put your assertion in an It().

48
vendor/github.com/onsi/gomega/internal/gutil/post_ioutil.go generated vendored Normal file
View File

@ -0,0 +1,48 @@
//go:build go1.16
// +build go1.16
// Package gutil is a replacement for ioutil, which should not be used in new
// code as of Go 1.16. With Go 1.16 and higher, this implementation
// uses the ioutil replacement functions in "io" and "os" with some
// Gomega specifics. This means that we should not get deprecation warnings
// for ioutil when they are added.
package gutil
import (
"io"
"os"
)
func NopCloser(r io.Reader) io.ReadCloser {
return io.NopCloser(r)
}
func ReadAll(r io.Reader) ([]byte, error) {
return io.ReadAll(r)
}
func ReadDir(dirname string) ([]string, error) {
entries, err := os.ReadDir(dirname)
if err != nil {
return nil, err
}
var names []string
for _, entry := range entries {
names = append(names, entry.Name())
}
return names, nil
}
func ReadFile(filename string) ([]byte, error) {
return os.ReadFile(filename)
}
func MkdirTemp(dir, pattern string) (string, error) {
return os.MkdirTemp(dir, pattern)
}
func WriteFile(filename string, data []byte) error {
return os.WriteFile(filename, data, 0644)
}

47
vendor/github.com/onsi/gomega/internal/gutil/using_ioutil.go generated vendored Normal file
View File

@ -0,0 +1,47 @@
//go:build !go1.16
// +build !go1.16
// Package gutil is a replacement for ioutil, which should not be used in new
// code as of Go 1.16. With Go 1.15 and lower, this implementation
// uses the ioutil functions, meaning that although Gomega is not officially
// supported on these versions, it is still likely to work.
package gutil
import (
"io"
"io/ioutil"
)
func NopCloser(r io.Reader) io.ReadCloser {
return ioutil.NopCloser(r)
}
func ReadAll(r io.Reader) ([]byte, error) {
return ioutil.ReadAll(r)
}
func ReadDir(dirname string) ([]string, error) {
files, err := ioutil.ReadDir(dirname)
if err != nil {
return nil, err
}
var names []string
for _, file := range files {
names = append(names, file.Name())
}
return names, nil
}
func ReadFile(filename string) ([]byte, error) {
return ioutil.ReadFile(filename)
}
func MkdirTemp(dir, pattern string) (string, error) {
return ioutil.TempDir(dir, pattern)
}
func WriteFile(filename string, data []byte) error {
return ioutil.WriteFile(filename, data, 0644)
}

26
vendor/github.com/onsi/gomega/matchers.go generated vendored
View File

@ -357,12 +357,12 @@ func HaveKeyWithValue(key interface{}, value interface{}) types.GomegaMatcher {
// type Person struct {
// FirstName string
// LastName string
// DOB time.Time
// DOB time.Time
// }
// Expect(book).To(HaveField("Title", "Les Miserables"))
// Expect(book).To(HaveField("Title", ContainSubstring("Les"))
// Expect(book).To(HaveField("Person.FirstName", Equal("Victor"))
// Expect(book).To(HaveField("Person.DOB.Year()", BeNumerically("<", 1900))
// Expect(book).To(HaveField("Author.FirstName", Equal("Victor"))
// Expect(book).To(HaveField("Author.DOB.Year()", BeNumerically("<", 1900))
func HaveField(field string, expected interface{}) types.GomegaMatcher {
return &matchers.HaveFieldMatcher{
Field: field,
@ -370,6 +370,26 @@ func HaveField(field string, expected interface{}) types.GomegaMatcher {
}
}
// HaveValue applies the given matcher to the value of actual, optionally and
// repeatedly dereferencing pointers or taking the concrete value of interfaces.
// Thus, the matcher will always be applied to non-pointer and non-interface
// values only. HaveValue will fail with an error if a pointer or interface is
// nil. It will also fail for more than 31 pointer or interface dereferences to
// guard against mistakenly applying it to arbitrarily deep linked pointers.
//
// HaveValue differs from gstruct.PointTo in that it does not expect actual to
// be a pointer (as gstruct.PointTo does) but instead also accepts non-pointer
// and even interface values.
//
// actual := 42
// Expect(actual).To(HaveValue(42))
// Expect(&actual).To(HaveValue(42))
func HaveValue(matcher types.GomegaMatcher) types.GomegaMatcher {
return &matchers.HaveValueMatcher{
Matcher: matcher,
}
}
//BeNumerically performs numerical assertions in a type-agnostic way.
//Actual and expected should be numbers, though the specific type of
//number is irrelevant (float32, float64, uint8, etc...).

4
vendor/github.com/onsi/gomega/matchers/have_http_body_matcher.go generated vendored
View File

@ -2,11 +2,11 @@ package matchers
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"github.com/onsi/gomega/format"
"github.com/onsi/gomega/internal/gutil"
"github.com/onsi/gomega/types"
)
@ -81,7 +81,7 @@ func (matcher *HaveHTTPBodyMatcher) body(actual interface{}) ([]byte, error) {
if a.Body != nil {
defer a.Body.Close()
var err error
matcher.cachedBody, err = io.ReadAll(a.Body)
matcher.cachedBody, err = gutil.ReadAll(a.Body)
if err != nil {
return nil, fmt.Errorf("error reading response body: %w", err)
}

4
vendor/github.com/onsi/gomega/matchers/have_http_status_matcher.go generated vendored
View File

@ -2,13 +2,13 @@ package matchers
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"github.com/onsi/gomega/format"
"github.com/onsi/gomega/internal/gutil"
)
type HaveHTTPStatusMatcher struct {
@ -78,7 +78,7 @@ func formatHttpResponse(input interface{}) string {
body := "<nil>"
if resp.Body != nil {
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
data, err := gutil.ReadAll(resp.Body)
if err != nil {
data = []byte("<error reading body>")
}

54
vendor/github.com/onsi/gomega/matchers/have_value.go generated vendored Normal file
View File

@ -0,0 +1,54 @@
package matchers
import (
"errors"
"reflect"
"github.com/onsi/gomega/format"
"github.com/onsi/gomega/types"
)
const maxIndirections = 31
type HaveValueMatcher struct {
Matcher types.GomegaMatcher // the matcher to apply to the "resolved" actual value.
resolvedActual interface{} // the ("resolved") value.
}
func (m *HaveValueMatcher) Match(actual interface{}) (bool, error) {
val := reflect.ValueOf(actual)
for allowedIndirs := maxIndirections; allowedIndirs > 0; allowedIndirs-- {
// return an error if value isn't valid. Please note that we cannot
// check for nil here, as we might not deal with a pointer or interface
// at this point.
if !val.IsValid() {
return false, errors.New(format.Message(
actual, "not to be <nil>"))
}
switch val.Kind() {
case reflect.Ptr, reflect.Interface:
// resolve pointers and interfaces to their values, then rinse and
// repeat.
if val.IsNil() {
return false, errors.New(format.Message(
actual, "not to be <nil>"))
}
val = val.Elem()
continue
default:
// forward the final value to the specified matcher.
m.resolvedActual = val.Interface()
return m.Matcher.Match(m.resolvedActual)
}
}
// too many indirections: extreme star gazing, indeed...?
return false, errors.New(format.Message(actual, "too many indirections"))
}
func (m *HaveValueMatcher) FailureMessage(_ interface{}) (message string) {
return m.Matcher.FailureMessage(m.resolvedActual)
}
func (m *HaveValueMatcher) NegatedFailureMessage(_ interface{}) (message string) {
return m.Matcher.NegatedFailureMessage(m.resolvedActual)
}

8
vendor/github.com/onsi/gomega/tools generated vendored Normal file
View File

@ -0,0 +1,8 @@
//go:build tools
// +build tools
package main
import (
_ "github.com/onsi/ginkgo/v2/ginkgo"
)

8
vendor/github.com/prometheus/client_golang/prometheus/collector.go generated vendored
View File

@ -118,3 +118,11 @@ func (c *selfCollector) Describe(ch chan<- *Desc) {
func (c *selfCollector) Collect(ch chan<- Metric) {
ch <- c.self
}
// collectorMetric is a metric that is also a collector.
// Because of selfCollector, most (if not all) Metrics in
// this package are also collectors.
type collectorMetric interface {
Metric
Collector
}

94
vendor/github.com/prometheus/client_golang/prometheus/go_collector_go117.go generated vendored
View File

@ -20,6 +20,7 @@ import (
"math"
"runtime"
"runtime/metrics"
"strings"
"sync"
//nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility.
@ -31,10 +32,14 @@ import (
type goCollector struct {
base baseGoCollector
// mu protects updates to all fields ensuring a consistent
// snapshot is always produced by Collect.
mu sync.Mutex
// rm... fields all pertain to the runtime/metrics package.
rmSampleBuf []metrics.Sample
rmSampleMap map[string]*metrics.Sample
rmMetrics []Metric
rmMetrics []collectorMetric
// With Go 1.17, the runtime/metrics package was introduced.
// From that point on, metric names produced by the runtime/metrics
@ -52,13 +57,24 @@ type goCollector struct {
// Deprecated: Use collectors.NewGoCollector instead.
func NewGoCollector() Collector {
descriptions := metrics.All()
descMap := make(map[string]*metrics.Description)
for i := range descriptions {
descMap[descriptions[i].Name] = &descriptions[i]
// Collect all histogram samples so that we can get their buckets.
// The API guarantees that the buckets are always fixed for the lifetime
// of the process.
var histograms []metrics.Sample
for _, d := range descriptions {
if d.Kind == metrics.KindFloat64Histogram {
histograms = append(histograms, metrics.Sample{Name: d.Name})
}
}
metrics.Read(histograms)
bucketsMap := make(map[string][]float64)
for i := range histograms {
bucketsMap[histograms[i].Name] = histograms[i].Value.Float64Histogram().Buckets
}
// Generate a Desc and ValueType for each runtime/metrics metric.
metricSet := make([]Metric, 0, len(descriptions))
metricSet := make([]collectorMetric, 0, len(descriptions))
sampleBuf := make([]metrics.Sample, 0, len(descriptions))
sampleMap := make(map[string]*metrics.Sample, len(descriptions))
for i := range descriptions {
@ -76,9 +92,10 @@ func NewGoCollector() Collector {
sampleBuf = append(sampleBuf, metrics.Sample{Name: d.Name})
sampleMap[d.Name] = &sampleBuf[len(sampleBuf)-1]
var m Metric
var m collectorMetric
if d.Kind == metrics.KindFloat64Histogram {
_, hasSum := rmExactSumMap[d.Name]
unit := d.Name[strings.IndexRune(d.Name, ':')+1:]
m = newBatchHistogram(
NewDesc(
BuildFQName(namespace, subsystem, name),
@ -86,6 +103,7 @@ func NewGoCollector() Collector {
nil,
nil,
),
internal.RuntimeMetricsBucketsForUnit(bucketsMap[d.Name], unit),
hasSum,
)
} else if d.Cumulative {
@ -130,9 +148,25 @@ func (c *goCollector) Collect(ch chan<- Metric) {
// Collect base non-memory metrics.
c.base.Collect(ch)
// Collect must be thread-safe, so prevent concurrent use of
// rmSampleBuf. Just read into rmSampleBuf but write all the data
// we get into our Metrics or MemStats.
//
// This lock also ensures that the Metrics we send out are all from
// the same updates, ensuring their mutual consistency insofar as
// is guaranteed by the runtime/metrics package.
//
// N.B. This locking is heavy-handed, but Collect is expected to be called
// relatively infrequently. Also the core operation here, metrics.Read,
// is fast (O(tens of microseconds)) so contention should certainly be
// low, though channel operations and any allocations may add to that.
c.mu.Lock()
defer c.mu.Unlock()
// Populate runtime/metrics sample buffer.
metrics.Read(c.rmSampleBuf)
// Update all our metrics from rmSampleBuf.
for i, sample := range c.rmSampleBuf {
// N.B. switch on concrete type because it's significantly more efficient
// than checking for the Counter and Gauge interface implementations. In
@ -157,7 +191,6 @@ func (c *goCollector) Collect(ch chan<- Metric) {
panic("unexpected metric type")
}
}
// ms is a dummy MemStats that we populate ourselves so that we can
// populate the old metrics from it.
var ms runtime.MemStats
@ -280,13 +313,27 @@ type batchHistogram struct {
// but Write calls may operate concurrently with updates.
// Contention between these two sources should be rare.
mu sync.Mutex
buckets []float64 // Inclusive lower bounds.
buckets []float64 // Inclusive lower bounds, like runtime/metrics.
counts []uint64
sum float64 // Used if hasSum is true.
}
func newBatchHistogram(desc *Desc, hasSum bool) *batchHistogram {
h := &batchHistogram{desc: desc, hasSum: hasSum}
// newBatchHistogram creates a new batch histogram value with the given
// Desc, buckets, and whether or not it has an exact sum available.
//
// buckets must always be from the runtime/metrics package, following
// the same conventions.
func newBatchHistogram(desc *Desc, buckets []float64, hasSum bool) *batchHistogram {
h := &batchHistogram{
desc: desc,
buckets: buckets,
// Because buckets follows runtime/metrics conventions, there's
// 1 more value in the buckets list than there are buckets represented,
// because in runtime/metrics, the bucket values represent *boundaries*,
// and non-Inf boundaries are inclusive lower bounds for that bucket.
counts: make([]uint64, len(buckets)-1),
hasSum: hasSum,
}
h.init(h)
return h
}
@ -294,28 +341,25 @@ func newBatchHistogram(desc *Desc, hasSum bool) *batchHistogram {
// update updates the batchHistogram from a runtime/metrics histogram.
//
// sum must be provided if the batchHistogram was created to have an exact sum.
// h.buckets must be a strict subset of his.Buckets.
func (h *batchHistogram) update(his *metrics.Float64Histogram, sum float64) {
counts, buckets := his.Counts, his.Buckets
// Skip a -Inf bucket altogether. It's not clear how to represent that.
if math.IsInf(buckets[0], -1) {
buckets = buckets[1:]
counts = counts[1:]
}
h.mu.Lock()
defer h.mu.Unlock()
// Check if we're initialized.
if h.buckets == nil {
// Make copies of counts and buckets. It's really important
// that we don't retain his.Counts or his.Buckets anywhere since
// it's going to get reused.
h.buckets = make([]float64, len(buckets))
copy(h.buckets, buckets)
h.counts = make([]uint64, len(counts))
// Clear buckets.
for i := range h.counts {
h.counts[i] = 0
}
// Copy and reduce buckets.
var j int
for i, count := range counts {
h.counts[j] += count
if buckets[i+1] == h.buckets[j+1] {
j++
}
}
copy(h.counts, counts)
if h.hasSum {
h.sum = sum
}

65
vendor/github.com/prometheus/client_golang/prometheus/internal/go_runtime_metrics.go generated vendored
View File

@ -17,6 +17,7 @@
package internal
import (
"math"
"path"
"runtime/metrics"
"strings"
@ -75,3 +76,67 @@ func RuntimeMetricsToProm(d *metrics.Description) (string, string, string, bool)
}
return namespace, subsystem, name, valid
}
// RuntimeMetricsBucketsForUnit takes a set of buckets obtained for a runtime/metrics histogram
// type (so, lower-bound inclusive) and a unit from a runtime/metrics name, and produces
// a reduced set of buckets. This function always removes any -Inf bucket as it's represented
// as the bottom-most upper-bound inclusive bucket in Prometheus.
func RuntimeMetricsBucketsForUnit(buckets []float64, unit string) []float64 {
switch unit {
case "bytes":
// Rebucket as powers of 2.
return rebucketExp(buckets, 2)
case "seconds":
// Rebucket as powers of 10 and then merge all buckets greater
// than 1 second into the +Inf bucket.
b := rebucketExp(buckets, 10)
for i := range b {
if b[i] <= 1 {
continue
}
b[i] = math.Inf(1)
b = b[:i+1]
break
}
return b
}
return buckets
}
// rebucketExp takes a list of bucket boundaries (lower bound inclusive) and
// downsamples the buckets to those a multiple of base apart. The end result
// is a roughly exponential (in many cases, perfectly exponential) bucketing
// scheme.
func rebucketExp(buckets []float64, base float64) []float64 {
bucket := buckets[0]
var newBuckets []float64
// We may see a -Inf here, in which case, add it and skip it
// since we risk producing NaNs otherwise.
//
// We need to preserve -Inf values to maintain runtime/metrics
// conventions. We'll strip it out later.
if bucket == math.Inf(-1) {
newBuckets = append(newBuckets, bucket)
buckets = buckets[1:]
bucket = buckets[0]
}
// From now on, bucket should always have a non-Inf value because
// Infs are only ever at the ends of the bucket lists, so
// arithmetic operations on it are non-NaN.
for i := 1; i < len(buckets); i++ {
if bucket >= 0 && buckets[i] < bucket*base {
// The next bucket we want to include is at least bucket*base.
continue
} else if bucket < 0 && buckets[i] < bucket/base {
// In this case the bucket we're targeting is negative, and since
// we're ascending through buckets here, we need to divide to get
// closer to zero exponentially.
continue
}
// The +Inf bucket will always be the last one, and we'll always
// end up including it here because bucket
newBuckets = append(newBuckets, bucket)
bucket = buckets[i]
}
return append(newBuckets, bucket)
}

5
vendor/modules.txt vendored
View File

@ -362,11 +362,12 @@ github.com/onsi/ginkgo/reporters/stenographer
github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable
github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty
github.com/onsi/ginkgo/types
# github.com/onsi/gomega v1.17.0
# github.com/onsi/gomega v1.18.0
## explicit; go 1.16
github.com/onsi/gomega
github.com/onsi/gomega/format
github.com/onsi/gomega/internal
github.com/onsi/gomega/internal/gutil
github.com/onsi/gomega/matchers
github.com/onsi/gomega/matchers/support/goraph/bipartitegraph
github.com/onsi/gomega/matchers/support/goraph/edge
@ -400,7 +401,7 @@ github.com/pkg/errors
# github.com/pmezard/go-difflib v1.0.0
## explicit
github.com/pmezard/go-difflib/difflib
# github.com/prometheus/client_golang v1.12.0
# github.com/prometheus/client_golang v1.12.1
## explicit; go 1.13
github.com/prometheus/client_golang/prometheus
github.com/prometheus/client_golang/prometheus/collectors