mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-26 00:00:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup: remove thick provisioning code

Browse Source 
This commit removes the thick provisioning
code as thick provisioning is deprecated in
cephcsi 3.5.0.

fixes: #2795

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2022-01-25 13:07:15 +05:30 committed by mergify[bot]
parent 4ee4fdfebd
commit 28fef9b379
14 changed files with 53 additions and 908 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
charts/ceph-csi-rbd/README.md
View File

@ -149,7 +149,6 @@ charts and their default values.
| `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `<cluster-ID>` | | `storageClass.clusterID` | String representing a Ceph cluster to provision storage from | `<cluster-ID>` |
| `storageClass.dataPool` | Specifies the erasure coded pool | `""` | | `storageClass.dataPool` | Specifies the erasure coded pool | `""` |
| `storageClass.pool` | Ceph pool into which the RBD image shall be created | `replicapool` | | `storageClass.pool` | Ceph pool into which the RBD image shall be created | `replicapool` |
| `storageClass.thickProvision` | Specifies whether thick provision should be enabled | `false` |
| `storageclass.imageFeatures` | Specifies RBD image features | `layering` | | `storageclass.imageFeatures` | Specifies RBD image features | `layering` |
| `storageclass.tryOtherMounters` | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason | `false` | | `storageclass.tryOtherMounters` | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason | `false` |
| `storageClass.mounter` | Specifies RBD mounter | `""` | | `storageClass.mounter` | Specifies RBD mounter | `""` |

1
charts/ceph-csi-rbd/templates/storageclass.yaml
View File

@ -18,7 +18,6 @@ parameters:
clusterID: {{ .Values.storageClass.clusterID }} clusterID: {{ .Values.storageClass.clusterID }}
pool: {{ .Values.storageClass.pool }} pool: {{ .Values.storageClass.pool }}
imageFeatures: {{ .Values.storageClass.imageFeatures }} imageFeatures: {{ .Values.storageClass.imageFeatures }}
thickProvision: {{ .Values.storageClass.thickProvision | quote}}
{{- if .Values.storageClass.tryOtherMounters }} {{- if .Values.storageClass.tryOtherMounters }}
tryOtherMounters: {{ .Values.storageClass.tryOtherMounters | quote}} tryOtherMounters: {{ .Values.storageClass.tryOtherMounters | quote}}
{{- end }} {{- end }}

4
charts/ceph-csi-rbd/values.yaml
View File

@ -279,10 +279,6 @@ storageClass:
# eg: pool: replicapool # eg: pool: replicapool
pool: replicapool pool: replicapool
# Set thickProvision to true if you want RBD images to be fully allocated on
# creation (thin provisioning is the default).
thickProvision: false
# (required) RBD image features, CSI creates image with image-format 2 # (required) RBD image features, CSI creates image with image-format 2
# CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
# `object-map`, `fast-diff` features. If `journaling` is enabled, must # `object-map`, `fast-diff` features. If `journaling` is enabled, must

35
docs/deploy-rbd.md
View File

@ -26,8 +26,8 @@ make image-cephcsi
**Available command line arguments:** **Available command line arguments:**
| Option | Default value | Description | | Option | Default value | Description |
| ------------------------ | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | ------------------------ | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `--endpoint` | `unix:///tmp/csi.sock` | CSI endpoint, must be a UNIX socket | | `--endpoint` | `unix:///tmp/csi.sock` | CSI endpoint, must be a UNIX socket |
| `--csi-addons-endpoint` | `unix:///tmp/csi-addons.sock` | CSI-Addons endpoint, must be a UNIX socket | | `--csi-addons-endpoint` | `unix:///tmp/csi-addons.sock` | CSI-Addons endpoint, must be a UNIX socket |
| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | | `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) |
@ -49,23 +49,22 @@ make image-cephcsi
**Available volume parameters:** **Available volume parameters:**
| Parameter | Required | Description | | Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | --------------------------------------------------------------------------------------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | | `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `pool` | yes | Ceph pool into which the RBD image shall be created | | `pool` | yes | Ceph pool into which the RBD image shall be created |
| `dataPool` | no | Ceph pool used for the data of the RBD images. | | `dataPool` | no | Ceph pool used for the data of the RBD images. |
| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). | | `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). |
| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). | | `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). |
| `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. | | `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, `object-map`, `fast-diff` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. |
| `tryOtherMounters` | no | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason | | `tryOtherMounters` | no | Specifies whether to try other mounters in case if the current mounter fails to mount the rbd image for any reason |
| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | | `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | | `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | | `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects |
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | | `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** | | `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** |
| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases | | `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases |
| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it (**DEPRECATED**: recommended alternative solution is to use accounting/quotas for created volumes) |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the **NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration) running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)

189
e2e/rbd.go
View File

@ -1747,38 +1747,6 @@ var _ = Describe("RBD", func() {
f) f)
}) })
By("create a thick-provisioned PVC-PVC clone and bind it to an app", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{
"thickProvision": "true",
}, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
validatePVCClone(1,
pvcPath,
appPath,
pvcSmartClonePath,
appSmartClonePath,
noDataPool,
noKMS,
isThickPVC,
f)
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create an encrypted PVC snapshot and restore it for an app with VaultKMS", func() { By("create an encrypted PVC snapshot and restore it for an app with VaultKMS", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml") err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil { if err != nil {
@ -1863,75 +1831,6 @@ var _ = Describe("RBD", func() {
} }
}) })
By("Validate thick PVC restore from vaultKMS to userSecretsMetadataKMS", func() {
restoreSCName := "restore-sc"
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
scOpts := map[string]string{
"encrypted": "true",
"encryptionKMSID": "vault-test",
"thickProvision": "true",
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, scOpts, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
scOpts = map[string]string{
"encrypted": "true",
"encryptionKMSID": "user-secrets-metadata-test",
"thickProvision": "true",
}
err = createRBDStorageClass(f.ClientSet, f, restoreSCName, nil, scOpts, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
// PVC creation namespace where secret will be created
namespace := f.UniqueName
// create user Secret
err = retryKubectlFile(namespace, kubectlCreate, vaultExamplePath+vaultUserSecret, deployTimeout)
if err != nil {
e2elog.Failf("failed to create user Secret: %v", err)
}
validatePVCSnapshot(1,
pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath,
vaultKMS, secretsMetadataKMS,
restoreSCName, noDataPool, f)
// delete user secret
err = retryKubectlFile(namespace,
kubectlDelete,
vaultExamplePath+vaultUserSecret,
deployTimeout,
"--ignore-not-found=true")
if err != nil {
e2elog.Failf("failed to delete user Secret: %v", err)
}
err = retryKubectlArgs(cephCSINamespace, kubectlDelete, deployTimeout, "storageclass", restoreSCName)
if err != nil {
e2elog.Failf("failed to delete storageclass %q: %v", restoreSCName, err)
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create an encrypted PVC-PVC clone and bind it to an app", func() { By("create an encrypted PVC-PVC clone and bind it to an app", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml") err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil { if err != nil {
@ -3180,47 +3079,6 @@ var _ = Describe("RBD", func() {
validateRBDImageCount(f, 0, defaultRBDPool) validateRBDImageCount(f, 0, defaultRBDPool)
}) })
By("create a thick-provisioned PVC", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, map[string]string{
"thickProvision": "true",
}, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
pvc, err := loadPVC(rawPvcPath)
if err != nil {
e2elog.Failf("failed to load PVC:: %v", err)
}
pvcSizes := []string{
// original value from the yaml file (100MB)
"100Mi",
// half the size (50MB), is not stripe-size roundable
"50Mi",
}
for _, pvcSize := range pvcSizes {
err = validateThickPVC(f, pvc, pvcSize)
if err != nil {
e2elog.Failf("validating thick-provisioning failed: %v", err)
}
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass: %v", err)
}
})
By("create a PVC and Bind it to an app for mapped rbd image with options", func() { By("create a PVC and Bind it to an app for mapped rbd image with options", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml") err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil { if err != nil {
@ -3270,53 +3128,6 @@ var _ = Describe("RBD", func() {
} }
}) })
By("validate the functionality of controller with encryption and thick-provisioning", func() {
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
e2elog.Failf("failed to delete storageclass : %v", err)
}
scParams := map[string]string{
"encrypted": "true",
"encryptionKMSID": "user-secrets-metadata-test",
"thickProvision": "true",
}
// PVC creation namespace where secret will be created
namespace := f.UniqueName
// create user Secret
err = retryKubectlFile(namespace, kubectlCreate, vaultExamplePath+vaultUserSecret, deployTimeout)
if err != nil {
e2elog.Failf("failed to create user Secret: %v", err)
}
err = validateController(f,
pvcPath, appPath, rbdExamplePath+"storageclass.yaml",
nil,
scParams)
if err != nil {
e2elog.Failf("failed to validate controller : %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
// delete user secret
err = retryKubectlFile(namespace,
kubectlDelete,
vaultExamplePath+vaultUserSecret,
deployTimeout,
"--ignore-not-found=true")
if err != nil {
e2elog.Failf("failed to delete user Secret: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
e2elog.Failf("failed to create storageclass : %v", err)
}
})
By("validate image deletion when it is moved to trash", func() { By("validate image deletion when it is moved to trash", func() {
// make sure pool is empty // make sure pool is empty
validateRBDImageCount(f, 0, defaultRBDPool) validateRBDImageCount(f, 0, defaultRBDPool)

104
e2e/rbd_helper.go
View File

@ -31,7 +31,6 @@ import (
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
scv1 "k8s.io/api/storage/v1" scv1 "k8s.io/api/storage/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes"
@ -39,11 +38,6 @@ import (
e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2elog "k8s.io/kubernetes/test/e2e/framework/log"
) )
const (
// image metadata key for thick-provisioning.
thickProvisionMetaKey = "rbd.csi.ceph.com/thick-provisioned"
)
// nolint:gomnd // numbers specify Kernel versions. // nolint:gomnd // numbers specify Kernel versions.
var nbdResizeSupport = []util.KernelVersion{ var nbdResizeSupport = []util.KernelVersion{
{ {
@ -529,39 +523,6 @@ func isEncryptedPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *
return validateEncryptedImage(f, rbdImageSpec, imageData.pvName, app.Name) return validateEncryptedImage(f, rbdImageSpec, imageData.pvName, app.Name)
} }
func isThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error {
du, err := getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize == 0 || du.UsedSize != du.ProvisionedSize {
return fmt.Errorf("backing RBD image is not thick-provisioned (%d/%d)", du.UsedSize, du.ProvisionedSize)
}
err = validateThickImageMetadata(f, pvc, thickProvisionMetaKey)
if err != nil {
return fmt.Errorf("failed to validate thick image: %w", err)
}
return nil
}
// validateThickImage check thick metadata is set on the the image.
func validateThickImageMetadata(f *framework.Framework, pvc *v1.PersistentVolumeClaim, key string) error {
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
thickState, err := getImageMeta(rbdImageSpec, key, f)
if err != nil {
return err
}
if thickState == "" {
return fmt.Errorf("image metadata is set for %s", rbdImageSpec)
}
return nil
}
// validateEncryptedImage verifies that the RBD image is encrypted. The // validateEncryptedImage verifies that the RBD image is encrypted. The
// following checks are performed: // following checks are performed:
// - Metadata of the image should be set with the encryption state; // - Metadata of the image should be set with the encryption state;
@ -630,6 +591,7 @@ func deleteBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim
return err return err
} }
//nolint:unused // required for reclaimspace e2e.
// rbdDuImage contains the disk-usage statistics of an RBD image. // rbdDuImage contains the disk-usage statistics of an RBD image.
type rbdDuImage struct { type rbdDuImage struct {
Name string `json:"name"` Name string `json:"name"`
@ -637,11 +599,13 @@ type rbdDuImage struct {
UsedSize uint64 `json:"used_size"` UsedSize uint64 `json:"used_size"`
} }
//nolint:unused // required for reclaimspace e2e.
// rbdDuImageList contains the list of images returned by 'rbd du'. // rbdDuImageList contains the list of images returned by 'rbd du'.
type rbdDuImageList struct { type rbdDuImageList struct {
Images []*rbdDuImage `json:"images"` Images []*rbdDuImage `json:"images"`
} }
//nolint:deadcode,unused // required for reclaimspace e2e.
// getRbdDu runs 'rbd du' on the RBD image and returns a rbdDuImage struct with // getRbdDu runs 'rbd du' on the RBD image and returns a rbdDuImage struct with
// the result. // the result.
func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImage, error) { func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImage, error) {
@ -672,6 +636,7 @@ func getRbdDu(f *framework.Framework, pvc *v1.PersistentVolumeClaim) (*rbdDuImag
return nil, fmt.Errorf("image %s not found", imageData.imageName) return nil, fmt.Errorf("image %s not found", imageData.imageName)
} }
//nolint:deadcode,unused // required for reclaimspace e2e.
// sparsifyBackingRBDImage runs `rbd sparsify` on the RBD image. Once done, all // sparsifyBackingRBDImage runs `rbd sparsify` on the RBD image. Once done, all
// data blocks that contain zeros are discarded/trimmed/unmapped and do not // data blocks that contain zeros are discarded/trimmed/unmapped and do not
// take up any space anymore. This can be used to verify that an empty, but // take up any space anymore. This can be used to verify that an empty, but
@ -903,67 +868,6 @@ func deletePVCCSIJournalInPool(f *framework.Framework, pvc *v1.PersistentVolumeC
return nil return nil
} }
func validateThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, size string) error {
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(size)
err := createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to create PVC: %w", err)
}
validateRBDImageCount(f, 1, defaultRBDPool)
err = validateThickImageMetadata(f, pvc, thickProvisionMetaKey)
if err != nil {
return fmt.Errorf("failed to validate thick image: %w", err)
}
// nothing has been written, but the image should be allocated
du, err := getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize == 0 || du.UsedSize != du.ProvisionedSize {
return fmt.Errorf("backing RBD image is not thick-provisioned (%d/%d)", du.UsedSize, du.ProvisionedSize)
}
// expanding the PVC should thick-allocate the expansion
// nolint:gomnd // we want 2x the size so that extending is done
newSize := du.ProvisionedSize * 2
err = expandPVCSize(f.ClientSet, pvc, fmt.Sprintf("%d", newSize), deployTimeout)
if err != nil {
return fmt.Errorf("failed to expand PVC: %w", err)
}
// after expansion, the updated 'du' should be larger
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("failed to get allocations of RBD image: %w", err)
} else if du.UsedSize != newSize {
return fmt.Errorf("backing RBD image is not extended thick-provisioned (%d/%d)", du.UsedSize, newSize)
}
// thick provisioning allows for sparsifying
err = sparsifyBackingRBDImage(f, pvc)
if err != nil {
return fmt.Errorf("failed to sparsify RBD image: %w", err)
}
// after sparsifying the image should not have any allocations
du, err = getRbdDu(f, pvc)
if err != nil {
return fmt.Errorf("backing RBD image is not thick-provisioned: %w", err)
} else if du.UsedSize != 0 {
return fmt.Errorf("backing RBD image was not sparsified (%d bytes allocated)", du.UsedSize)
}
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
return fmt.Errorf("failed to delete PVC:: %w", err)
}
validateRBDImageCount(f, 0, defaultRBDPool)
return nil
}
// trashInfo contains the image details in trash. // trashInfo contains the image details in trash.
type trashInfo struct { type trashInfo struct {
Name string `json:"name"` Name string `json:"name"`

4
examples/rbd/storageclass.yaml
View File

@ -29,10 +29,6 @@ parameters:
# eg: pool: rbdpool # eg: pool: rbdpool
pool: <rbd-pool-name> pool: <rbd-pool-name>
# Deprecated: Set thickProvision to true if you want RBD images to be fully
# allocated on creation (thin provisioning is the default).
# thickProvision: "false"
# (required) RBD image features, CSI creates image with image-format 2 # (required) RBD image features, CSI creates image with image-format 2
# CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`, # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`,
# `object-map`, `fast-diff` features. If `journaling` is enabled, must # `object-map`, `fast-diff` features. If `journaling` is enabled, must

45
internal/rbd/clone.go
View File

@ -167,13 +167,6 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol
} }
} }
if rv.ThickProvision {
err = rv.setThickProvisioned()
if err != nil {
return fmt.Errorf("failed mark %q thick-provisioned: %w", rv, err)
}
}
err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID) err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID)
if err != nil { if err != nil {
log.ErrorLog(ctx, "failed to store volume %s: %v", rv, err) log.ErrorLog(ctx, "failed to store volume %s: %v", rv, err)
@ -236,39 +229,27 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro
} }
}() }()
if rv.ThickProvision { // flatten clone
err = tempClone.DeepCopy(&rv.rbdImage) errFlatten = tempClone.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if err != nil { if errFlatten != nil {
return fmt.Errorf("failed to deep copy %q into %q: %w", parentVol, rv, err) return errFlatten
} }
} else {
// flatten clone
errFlatten = tempClone.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if errFlatten != nil {
return errFlatten
}
// create snap of temp clone from temporary cloned image // create snap of temp clone from temporary cloned image
// create final clone // create final clone
// delete snap of temp clone // delete snap of temp clone
errClone = createRBDClone(ctx, tempClone, rv, cloneSnap) errClone = createRBDClone(ctx, tempClone, rv, cloneSnap)
if errClone != nil { if errClone != nil {
// set errFlatten error to cleanup temporary snapshot and temporary clone // set errFlatten error to cleanup temporary snapshot and temporary clone
errFlatten = errors.New("failed to create user requested cloned image") errFlatten = errors.New("failed to create user requested cloned image")
return errClone return errClone
}
} }
return nil return nil
} }
func (rv *rbdVolume) flattenCloneImage(ctx context.Context) error { func (rv *rbdVolume) flattenCloneImage(ctx context.Context) error {
if rv.ThickProvision {
// thick-provisioned images do not need flattening
return nil
}
tempClone := rv.generateTempClone() tempClone := rv.generateTempClone()
// reducing the limit for cloned images to make sure the limit is in range, // reducing the limit for cloned images to make sure the limit is in range,
// If the intermediate clone reaches the depth we may need to return ABORT // If the intermediate clone reaches the depth we may need to return ABORT

203
internal/rbd/controllerserver.go
View File

@ -19,8 +19,6 @@ package rbd
import ( import (
"context" "context"
"errors" "errors"
"fmt"
"strconv"
csicommon "github.com/ceph/ceph-csi/internal/csi-common" csicommon "github.com/ceph/ceph-csi/internal/csi-common"
"github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util"
@ -132,8 +130,6 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error()) return nil, status.Error(codes.InvalidArgument, err.Error())
} }
rbdVol.ThickProvision = isThickProvisionRequest(req.GetParameters())
rbdVol.RequestName = req.GetName() rbdVol.RequestName = req.GetName()
// Volume Size - Default is 1 GiB // Volume Size - Default is 1 GiB
@ -211,11 +207,6 @@ func checkValidCreateVolumeRequest(rbdVol, parentVol *rbdVolume, rbdSnap *rbdSna
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error()) return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
} }
err = rbdSnap.isCompatibleThickProvision(rbdVol)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
}
err = rbdSnap.isCompabitableClone(&rbdVol.rbdImage) err = rbdSnap.isCompabitableClone(&rbdVol.rbdImage)
if err != nil { if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error()) return status.Errorf(codes.InvalidArgument, "cannot restore from snapshot %s: %s", rbdSnap, err.Error())
@ -227,11 +218,6 @@ func checkValidCreateVolumeRequest(rbdVol, parentVol *rbdVolume, rbdSnap *rbdSna
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error()) return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
} }
err = parentVol.isCompatibleThickProvision(rbdVol)
if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
}
err = parentVol.isCompabitableClone(&rbdVol.rbdImage) err = parentVol.isCompabitableClone(&rbdVol.rbdImage)
if err != nil { if err != nil {
return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error()) return status.Errorf(codes.InvalidArgument, "cannot clone from volume %s: %s", parentVol, err.Error())
@ -287,7 +273,7 @@ func (cs *ControllerServer) CreateVolume(
if err != nil { if err != nil {
return nil, getGRPCErrorForCreateVolume(err) return nil, getGRPCErrorForCreateVolume(err)
} else if found { } else if found {
return cs.repairExistingVolume(ctx, req, cr, rbdVol, parentVol, rbdSnap) return cs.repairExistingVolume(ctx, req, cr, rbdVol, rbdSnap)
} }
err = checkValidCreateVolumeRequest(rbdVol, parentVol, rbdSnap) err = checkValidCreateVolumeRequest(rbdVol, parentVol, rbdSnap)
@ -352,50 +338,12 @@ func flattenParentImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent
// that the state is corrected to what was requested. It is needed to call this // that the state is corrected to what was requested. It is needed to call this
// when the process of creating a volume was interrupted. // when the process of creating a volume was interrupted.
func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.CreateVolumeRequest, func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.CreateVolumeRequest,
cr *util.Credentials, rbdVol, parentVol *rbdVolume, rbdSnap *rbdSnapshot) (*csi.CreateVolumeResponse, error) { cr *util.Credentials, rbdVol *rbdVolume, rbdSnap *rbdSnapshot) (*csi.CreateVolumeResponse, error) {
vcs := req.GetVolumeContentSource() vcs := req.GetVolumeContentSource()
switch { switch {
// normal CreateVolume without VolumeContentSource
case vcs == nil:
// continue/restart allocating the volume in case it
// should be thick-provisioned
if isThickProvisionRequest(req.GetParameters()) {
err := rbdVol.RepairThickProvision()
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
}
// rbdVol is a restore from snapshot, rbdSnap is passed // rbdVol is a restore from snapshot, rbdSnap is passed
case vcs.GetSnapshot() != nil: case vcs.GetSnapshot() != nil:
// When restoring of a thick-provisioned volume was happening,
// the image should be marked as thick-provisioned, unless it
// was aborted in flight. In order to restart the
// thick-restoring, delete the volume and let the caller retry
// from the start.
if isThickProvisionRequest(req.GetParameters()) {
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(
codes.Aborted,
"failed to verify thick-provisioned volume %q: %s",
rbdVol,
err)
} else if !thick {
err = rbdVol.deleteImage(ctx)
if err != nil {
return nil, status.Errorf(codes.Aborted, "failed to remove partially cloned volume %q: %s", rbdVol, err)
}
err = undoVolReservation(ctx, rbdVol, cr)
if err != nil {
return nil, status.Errorf(codes.Aborted, "failed to remove volume %q from journal: %s", rbdVol, err)
}
return nil, status.Errorf(
codes.Aborted,
"restoring thick-provisioned volume %q has been interrupted, please retry", rbdVol)
}
}
// restore from snapshot implies rbdSnap != nil // restore from snapshot implies rbdSnap != nil
// check if image depth is reached limit and requires flatten // check if image depth is reached limit and requires flatten
err := checkFlatten(ctx, rbdVol, cr) err := checkFlatten(ctx, rbdVol, cr)
@ -418,23 +366,6 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
// rbdVol is a clone from parentVol // rbdVol is a clone from parentVol
case vcs.GetVolume() != nil: case vcs.GetVolume() != nil:
// When cloning into a thick-provisioned volume was happening,
// the image should be marked as thick-provisioned, unless it
// was aborted in flight. In order to restart the
// thick-cloning, delete the volume and undo the reservation in
// the journal to let the caller retry from the start.
if isThickProvisionRequest(req.GetParameters()) {
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(
codes.Internal,
"failed to verify thick-provisioned volume %q: %s",
rbdVol,
err)
} else if !thick {
return nil, cleanupThickClone(ctx, parentVol, rbdVol, rbdSnap, cr)
}
}
// expand the image if the requested size is greater than the current size // expand the image if the requested size is greater than the current size
err := rbdVol.expand() err := rbdVol.expand()
if err != nil { if err != nil {
@ -447,26 +378,6 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
return buildCreateVolumeResponse(req, rbdVol), nil return buildCreateVolumeResponse(req, rbdVol), nil
} }
// cleanupThickClone will delete the snapshot and volume and undo the reservation.
func cleanupThickClone(ctx context.Context,
rbdVol,
parentVol *rbdVolume,
rbdSnap *rbdSnapshot,
cr *util.Credentials) error {
err := cleanUpSnapshot(ctx, parentVol, rbdSnap, rbdVol)
if err != nil {
return status.Errorf(codes.Internal, "failed to remove partially cloned volume %q: %s", rbdVol, err)
}
err = undoVolReservation(ctx, rbdVol, cr)
if err != nil {
return status.Errorf(codes.Internal, "failed to remove volume %q from journal: %s", rbdVol, err)
}
return status.Errorf(
codes.Internal,
"cloning thick-provisioned volume %q has been interrupted, please retry", rbdVol)
}
// check snapshots on the rbd image, as we have limit from krbd that an image // check snapshots on the rbd image, as we have limit from krbd that an image
// cannot have more than 510 snapshot at a given point of time. If the // cannot have more than 510 snapshot at a given point of time. If the
// snapshots are more than the `maxSnapshotsOnImage` Add a task to flatten all // snapshots are more than the `maxSnapshotsOnImage` Add a task to flatten all
@ -474,11 +385,6 @@ func cleanupThickClone(ctx context.Context,
// are more than the `minSnapshotOnImage` Add a task to flatten all the // are more than the `minSnapshotOnImage` Add a task to flatten all the
// temporary cloned images. // temporary cloned images.
func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error { func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) error {
if rbdVol.ThickProvision {
// thick-provisioned images do not need flattening
return nil
}
snaps, err := rbdVol.listSnapshots() snaps, err := rbdVol.listSnapshots()
if err != nil { if err != nil {
if errors.Is(err, ErrImageNotFound) { if errors.Is(err, ErrImageNotFound) {
@ -592,27 +498,12 @@ func (cs *ControllerServer) createVolumeFromSnapshot(
// as we are operating on single cluster reuse the connection // as we are operating on single cluster reuse the connection
parentVol.conn = rbdVol.conn.Copy() parentVol.conn = rbdVol.conn.Copy()
if rbdVol.ThickProvision { // create clone image and delete snapshot
err = parentVol.DeepCopy(&rbdVol.rbdImage) err = rbdVol.cloneRbdImageFromSnapshot(ctx, rbdSnap, parentVol)
if err != nil { if err != nil {
return status.Errorf(codes.Internal, "failed to deep copy %q into %q: %v", parentVol, rbdVol, err) log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err)
}
err = rbdVol.setThickProvisioned()
if err != nil {
return status.Errorf(codes.Internal, "failed to mark %q thick-provisioned: %s", rbdVol, err)
}
err = parentVol.copyEncryptionConfig(&rbdVol.rbdImage, true)
if err != nil {
return status.Errorf(codes.Internal, err.Error())
}
} else {
// create clone image and delete snapshot
err = rbdVol.cloneRbdImageFromSnapshot(ctx, rbdSnap, parentVol)
if err != nil {
log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err)
return err return err
}
} }
log.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol, rbdSnap) log.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol, rbdSnap)
@ -1136,31 +1027,6 @@ func cloneFromSnapshot(
} }
} }
// The clone image created during CreateSnapshot has to be marked as thick.
// As snapshot and volume both are independent we cannot depend on the
// parent volume of the clone to check thick provision during CreateVolume
// from snapshot operation because the parent volume can be deleted anytime
// after snapshot is created.
// TODO: copy thick provision config
thick, err := rbdVol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed checking thick-provisioning of %q: %s", rbdVol, err)
}
if thick {
// check the thick metadata is already set on the clone image.
thick, err = vol.isThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed checking thick-provisioning of %q: %s", vol, err)
}
if !thick {
err = vol.setThickProvisioned()
if err != nil {
return nil, status.Errorf(codes.Internal, "failed mark %q thick-provisioned: %s", vol, err)
}
}
}
err = vol.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth) err = vol.flattenRbdImage(ctx, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth)
if errors.Is(err, ErrFlattenInProgress) { if errors.Is(err, ErrFlattenInProgress) {
// if flattening is in progress, return error and do not cleanup // if flattening is in progress, return error and do not cleanup
@ -1258,30 +1124,13 @@ func (cs *ControllerServer) doSnapshotClone(
} }
} }
// The clone image created during CreateSnapshot has to be marked as thick. err = cloneRbd.createSnapshot(ctx, rbdSnap)
// As snapshot and volume both are independent we cannot depend on the
// parent volume of the clone to check thick provision during CreateVolume
// from snapshot operation because the parent volume can be deleted anytime
// after snapshot is created.
thick, err := parentVol.isThickProvisioned()
if err != nil { if err != nil {
return nil, fmt.Errorf("failed checking thick-provisioning of %q: %w", parentVol, err) // update rbd image name for logging
} rbdSnap.RbdImageName = cloneRbd.RbdImageName
log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err)
if thick { return cloneRbd, err
err = cloneRbd.setThickProvisioned()
if err != nil {
return nil, fmt.Errorf("failed mark %q thick-provisioned: %w", cloneRbd, err)
}
} else {
err = cloneRbd.createSnapshot(ctx, rbdSnap)
if err != nil {
// update rbd image name for logging
rbdSnap.RbdImageName = cloneRbd.RbdImageName
log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err)
return cloneRbd, err
}
} }
err = cloneRbd.getImageID() err = cloneRbd.getImageID()
@ -1542,31 +1391,3 @@ func (cs *ControllerServer) ControllerExpandVolume(
NodeExpansionRequired: nodeExpansion, NodeExpansionRequired: nodeExpansion,
}, nil }, nil
} }
// logThickProvisioningDeprecation makes sure the deprecation warning about
// thick-provisining is logged only once.
var logThickProvisioningDeprecation = true
// isThickProvisionRequest returns true in case the request contains the
// `thickProvision` option set to `true`.
func isThickProvisionRequest(parameters map[string]string) bool {
tp := "thickProvision"
thick, ok := parameters[tp]
if !ok || thick == "" {
return false
}
thickBool, err := strconv.ParseBool(thick)
if err != nil {
return false
}
if logThickProvisioningDeprecation {
log.WarningLogMsg("thick-provisioning is deprecated and will " +
"be removed in a future release")
logThickProvisioningDeprecation = false
}
return thickBool
}

69
internal/rbd/controllerserver_test.go
View File

@ -1,69 +0,0 @@
/*
Copyright 2021 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"testing"
"github.com/container-storage-interface/spec/lib/go/csi"
)
func TestIsThickProvisionRequest(t *testing.T) {
t.Parallel()
req := &csi.CreateVolumeRequest{
Name: "fake",
Parameters: map[string]string{
"unkownOption": "not-set",
},
}
// pass disabled/invalid values for "thickProvision" option
if isThickProvisionRequest(req.GetParameters()) {
t.Error("request is not for thick-provisioning")
}
req.Parameters["thickProvision"] = ""
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "false"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "off"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "no"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
req.Parameters["thickProvision"] = "**true**"
if isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request is not for thick-provisioning: %s", req.Parameters["thickProvision"])
}
// only "true" should enable thick provisioning
req.Parameters["thickProvision"] = "true"
if !isThickProvisionRequest(req.GetParameters()) {
t.Errorf("request should be for thick-provisioning: %s", req.Parameters["thickProvision"])
}
}

1
internal/rbd/nodeserver.go
View File

@ -178,7 +178,6 @@ func populateRbdVol(
return nil, status.Error(codes.Internal, err.Error()) return nil, status.Error(codes.Internal, err.Error())
} }
rv.ThickProvision = isThickProvisionRequest(req.GetVolumeContext())
isStaticVol := parseBoolOption(ctx, req.GetVolumeContext(), staticVol, false) isStaticVol := parseBoolOption(ctx, req.GetVolumeContext(), staticVol, false)
// get rbd image name from the volume journal // get rbd image name from the volume journal
// for static volumes, the image name is actually the volume ID itself // for static volumes, the image name is actually the volume ID itself

37
internal/rbd/rbd_attach.go
View File

@ -310,7 +310,7 @@ func attachRBDImage(ctx context.Context, volOptions *rbdVolume, device string, c
return devicePath, err return devicePath, err
} }
func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions, cookie string) []string { func appendNbdDeviceTypeAndOptions(cmdArgs []string, userOptions, cookie string) []string {
cmdArgs = append(cmdArgs, "--device-type", accessTypeNbd) cmdArgs = append(cmdArgs, "--device-type", accessTypeNbd)
isUnmap := CheckSliceContains(cmdArgs, "unmap") isUnmap := CheckSliceContains(cmdArgs, "unmap")
@ -328,12 +328,6 @@ func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions,
if hasNBDCookieSupport { if hasNBDCookieSupport {
cmdArgs = append(cmdArgs, "--options", fmt.Sprintf("cookie=%s", cookie)) cmdArgs = append(cmdArgs, "--options", fmt.Sprintf("cookie=%s", cookie))
} }
if isThick {
// When an image is thick-provisioned, any discard/unmap/trim
// requests should not free extents.
cmdArgs = append(cmdArgs, "--options", "notrim")
}
} }
if userOptions != "" { if userOptions != "" {
@ -345,22 +339,11 @@ func appendNbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions,
return cmdArgs return cmdArgs
} }
func appendKRbdDeviceTypeAndOptions(cmdArgs []string, isThick bool, userOptions string) []string { func appendKRbdDeviceTypeAndOptions(cmdArgs []string, userOptions string) []string {
cmdArgs = append(cmdArgs, "--device-type", accessTypeKRbd)
isUnmap := CheckSliceContains(cmdArgs, "unmap")
if !isUnmap {
if isThick {
// When an image is thick-provisioned, any discard/unmap/trim
// requests should not free extents.
cmdArgs = append(cmdArgs, "--options", "notrim")
}
}
// Enable mapping and unmapping images from a non-initial network // Enable mapping and unmapping images from a non-initial network
// namespace (e.g. for Multus CNI). The network namespace must be // namespace (e.g. for Multus CNI). The network namespace must be
// owned by the initial user namespace. // owned by the initial user namespace.
cmdArgs = append(cmdArgs, "--options", "noudev") cmdArgs = append(cmdArgs, "--device-type", accessTypeKRbd, "--options", "noudev")
if userOptions != "" { if userOptions != "" {
// userOptions is appended after, possibly overriding the above // userOptions is appended after, possibly overriding the above
@ -413,12 +396,6 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util.
isNbd = true isNbd = true
} }
// check if the image should stay thick-provisioned
isThick, err := volOpt.isThickProvisioned()
if err != nil {
log.WarningLog(ctx, "failed to detect if image %q is thick-provisioned: %v", volOpt, err)
}
if isNbd { if isNbd {
mapArgs = append(mapArgs, "--log-file", mapArgs = append(mapArgs, "--log-file",
getCephClientLogFileName(volOpt.VolID, volOpt.LogDir, "rbd-nbd")) getCephClientLogFileName(volOpt.VolID, volOpt.LogDir, "rbd-nbd"))
@ -433,9 +410,9 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util.
} else { } else {
mapArgs = append(mapArgs, "map", imagePath) mapArgs = append(mapArgs, "map", imagePath)
if isNbd { if isNbd {
mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions, volOpt.VolID) mapArgs = appendNbdDeviceTypeAndOptions(mapArgs, volOpt.MapOptions, volOpt.VolID)
} else { } else {
mapArgs = appendKRbdDeviceTypeAndOptions(mapArgs, isThick, volOpt.MapOptions) mapArgs = appendKRbdDeviceTypeAndOptions(mapArgs, volOpt.MapOptions)
} }
} }
@ -543,9 +520,9 @@ func detachRBDImageOrDeviceSpec(
unmapArgs := []string{"unmap", dArgs.imageOrDeviceSpec} unmapArgs := []string{"unmap", dArgs.imageOrDeviceSpec}
if dArgs.isNbd { if dArgs.isNbd {
unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions, dArgs.volumeID) unmapArgs = appendNbdDeviceTypeAndOptions(unmapArgs, dArgs.unmapOptions, dArgs.volumeID)
} else { } else {
unmapArgs = appendKRbdDeviceTypeAndOptions(unmapArgs, false, dArgs.unmapOptions) unmapArgs = appendKRbdDeviceTypeAndOptions(unmapArgs, dArgs.unmapOptions)
} }
_, stderr, err := util.ExecCommand(ctx, rbd, unmapArgs...) _, stderr, err := util.ExecCommand(ctx, rbd, unmapArgs...)

248
internal/rbd/rbd_util.go
View File

@ -58,18 +58,6 @@ const (
rbdTaskRemoveCmdInvalidString = "No handler found" rbdTaskRemoveCmdInvalidString = "No handler found"
rbdTaskRemoveCmdAccessDeniedMessage = "access denied:" rbdTaskRemoveCmdAccessDeniedMessage = "access denied:"
// image metadata key for thick-provisioning.
// As image metadata key starting with '.rbd' will not be copied when we do
// clone or mirroring, deprecating the old key for the same reason use
// 'thickProvisionMetaKey' to set image metadata.
deprecatedthickProvisionMetaKey = ".rbd.csi.ceph.com/thick-provisioned"
thickProvisionMetaKey = "rbd.csi.ceph.com/thick-provisioned"
// these are the metadata set on the image to identify the image is
// thick provisioned or thin provisioned.
thickProvisionMetaData = "true"
thinProvisionMetaData = "false"
// migration label key and value for parameters in volume context. // migration label key and value for parameters in volume context.
intreeMigrationKey = "migration" intreeMigrationKey = "migration"
intreeMigrationLabel = "true" intreeMigrationLabel = "true"
@ -173,7 +161,6 @@ type rbdVolume struct {
RequestedVolSize int64 RequestedVolSize int64
DisableInUseChecks bool DisableInUseChecks bool
readOnly bool readOnly bool
ThickProvision bool
} }
// rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics. // rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics.
@ -382,26 +369,6 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er
} }
} }
if pOpts.ThickProvision {
err = pOpts.allocate(0)
if err != nil {
// nolint:errcheck // deleteImage() will log errors in
// case it fails, no need to log them here again
_ = pOpts.deleteImage(ctx)
return fmt.Errorf("failed to thick provision image: %w", err)
}
err = pOpts.setThickProvisioned()
if err != nil {
// nolint:errcheck // deleteImage() will log errors in
// case it fails, no need to log them here again
_ = pOpts.deleteImage(ctx)
return fmt.Errorf("failed to mark image as thick-provisioned: %w", err)
}
}
return nil return nil
} }
@ -465,82 +432,6 @@ func (ri *rbdImage) open() (*librbd.Image, error) {
return image, nil return image, nil
} }
// allocate uses the stripe-period of the image to fully allocate (thick
// provision) the image.
func (ri *rbdImage) allocate(offset uint64) error {
// We do not want to call discard, we really want to write zeros to get
// the allocation. This sets the option for the re-used connection, and
// all subsequent images that are opened. That is not a problem, as
// this is the only place images get written.
err := ri.conn.DisableDiscardOnZeroedWriteSame()
if err != nil {
return err
}
image, err := ri.open()
if err != nil {
return err
}
defer image.Close()
st, err := image.Stat()
if err != nil {
return err
}
sc, err := image.GetStripeCount()
if err != nil {
return err
}
// blockSize is the stripe-period: size of the object-size multiplied
// by the stripe-count
blockSize := sc * (1 << st.Order)
zeroBlock := make([]byte, blockSize)
// the actual size of the image as available in the pool, can be
// marginally different from the requested image size
size := st.Size - offset
// In case the remaining space on the volume is smaller than blockSize,
// write a partial block with WriteAt() after this loop.
for size > blockSize {
writeSize := size
// write a maximum of 1GB per WriteSame() call
if size > helpers.GiB {
writeSize = helpers.GiB
}
// round down to the size of a zeroBlock
if (writeSize % blockSize) != 0 {
writeSize = (writeSize / blockSize) * blockSize
}
_, err = image.WriteSame(offset, writeSize, zeroBlock,
rados.OpFlagNone)
if err != nil {
return fmt.Errorf("failed to allocate %d/%d bytes at "+
"offset %d: %w", writeSize, blockSize, offset, err)
}
// write succeeded
size -= writeSize
offset += writeSize
}
// write the last remaining bytes, in case the image size can not be
// written with the optimal blockSize
if size != 0 {
_, err = image.WriteAt(zeroBlock[:size], int64(offset))
if err != nil {
return fmt.Errorf("failed to allocate %d bytes at "+
"offset %d: %w", size, offset, err)
}
}
return nil
}
// isInUse checks if there is a watcher on the image. It returns true if there // isInUse checks if there is a watcher on the image. It returns true if there
// is a watcher on the image, otherwise returns false. // is a watcher on the image, otherwise returns false.
func (ri *rbdImage) isInUse() (bool, error) { func (ri *rbdImage) isInUse() (bool, error) {
@ -1710,40 +1601,10 @@ func (ri *rbdImage) resize(newSize int64) error {
} }
defer image.Close() defer image.Close()
thick, err := ri.isThickProvisioned()
if err != nil {
return err
}
// offset is used to track from where on the expansion is done, so that
// the extents can be allocated in case the image is thick-provisioned
var offset uint64
if thick {
st, statErr := image.Stat()
if statErr != nil {
return statErr
}
offset = st.Size
}
err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB)) err = image.Resize(uint64(util.RoundOffVolSize(newSize) * helpers.MiB))
if err != nil { if err != nil {
return err return err
} }
if thick {
err = ri.allocate(offset)
if err != nil {
resizeErr := image.Resize(offset)
if resizeErr != nil {
err = fmt.Errorf("failed to shrink image (%v) after failed allocation: %w", resizeErr, err)
}
return err
}
}
// update Volsize of rbdVolume object to newSize. // update Volsize of rbdVolume object to newSize.
ri.VolSize = newSize ri.VolSize = newSize
@ -1820,71 +1681,6 @@ func (ri *rbdImage) MigrateMetadata(oldKey, newKey, defaultValue string) (string
return value, nil return value, nil
} }
// setThickProvisioned records in the image metadata that it has been
// thick-provisioned.
func (ri *rbdImage) setThickProvisioned() error {
err := ri.SetMetadata(thickProvisionMetaKey, thickProvisionMetaData)
if err != nil {
return fmt.Errorf("failed to set metadata %q for %q: %w", thickProvisionMetaKey, ri, err)
}
return nil
}
// isThickProvisioned checks in the image metadata if the image has been marked
// as thick-provisioned. This can be used while expanding the image, so that
// the expansion can be allocated too.
func (ri *rbdImage) isThickProvisioned() (bool, error) {
value, err := ri.MigrateMetadata(deprecatedthickProvisionMetaKey, thickProvisionMetaKey, thinProvisionMetaData)
if err != nil {
return false, fmt.Errorf("failed to get metadata %q for %q: %w", thickProvisionMetaKey, ri, err)
}
thick, err := strconv.ParseBool(value)
if err != nil {
return false, fmt.Errorf("failed to convert %q=%q to a boolean: %w", thickProvisionMetaKey, value, err)
}
return thick, nil
}
// RepairThickProvision writes zero bytes to the volume so that it will be
// completely allocated. In case the volume is already marked as
// thick-provisioned, nothing will be done.
func (ri *rbdImage) RepairThickProvision() error {
// if the image has the thick-provisioned metadata, it has been fully
// allocated
done, err := ri.isThickProvisioned()
if err != nil {
return fmt.Errorf("failed to repair thick-provisioning of %q: %w", ri, err)
} else if done {
return nil
}
// in case there are watchers, assume allocating is still happening in
// the background (by an other process?)
background, err := ri.isInUse()
if err != nil {
return fmt.Errorf("failed to get users of %q: %w", ri, err)
} else if background {
return fmt.Errorf("not going to restart thick-provisioning of in-use image %q", ri)
}
// TODO: can this be improved by starting at the offset where
// allocating was aborted/restarted?
err = ri.allocate(0)
if err != nil {
return fmt.Errorf("failed to continue thick-provisioning of %q: %w", ri, err)
}
err = ri.setThickProvisioned()
if err != nil {
return fmt.Errorf("failed to continue thick-provisioning of %q: %w", ri, err)
}
return nil
}
// DeepCopy creates an independent image (dest) from the source image. This // DeepCopy creates an independent image (dest) from the source image. This
// process may take some time when the image is large. // process may take some time when the image is large.
func (ri *rbdImage) DeepCopy(dest *rbdImage) error { func (ri *rbdImage) DeepCopy(dest *rbdImage) error {
@ -2004,50 +1800,6 @@ func (ri *rbdImage) isCompabitableClone(dst *rbdImage) error {
return nil return nil
} }
func (ri *rbdImage) isCompatibleThickProvision(dst *rbdVolume) error {
thick, err := ri.isThickProvisioned()
if err != nil {
return err
}
switch {
case thick && !dst.ThickProvision:
return fmt.Errorf("cannot create thin volume from thick volume %q", ri)
case !thick && dst.ThickProvision:
return fmt.Errorf("cannot create thick volume from thin volume %q", ri)
}
return nil
}
// FIXME: merge isCompatibleThickProvision of rbdSnapshot and rbdImage to a single
// function.
func (rs *rbdSnapshot) isCompatibleThickProvision(dst *rbdVolume) error {
// During CreateSnapshot the rbd image will be created with the
// snapshot name. Replacing RbdImageName with RbdSnapName so that we
// can check if the image is thick provisioned
vol := generateVolFromSnap(rs)
err := vol.Connect(rs.conn.Creds)
if err != nil {
return err
}
defer vol.Destroy()
thick, err := vol.isThickProvisioned()
if err != nil {
return err
}
switch {
case thick && !dst.ThickProvision:
return fmt.Errorf("cannot create thin volume from thick volume %q", vol)
case !thick && dst.ThickProvision:
return fmt.Errorf("cannot create thick volume from thin volume %q", vol)
}
return nil
}
func (ri *rbdImage) addSnapshotScheduling( func (ri *rbdImage) addSnapshotScheduling(
interval admin.Interval, interval admin.Interval,
startTime admin.StartTime) error { startTime admin.StartTime) error {

20
internal/util/connection.go
View File

@ -140,23 +140,3 @@ func (cc *ClusterConnection) GetTaskAdmin() (*ra.TaskAdmin, error) {
return rbdAdmin.Task(), nil return rbdAdmin.Task(), nil
} }
// DisableDiscardOnZeroedWriteSame enables the
// `rbd_discard_on_zeroed_write_same` option in the cluster connection, so that
// writing zero blocks of data are actual writes on the OSDs (doing
// allocations) and not discard calls. This makes writes much slower, but
// enables the option to do thick-provisioning.
func (cc *ClusterConnection) DisableDiscardOnZeroedWriteSame() error {
if cc.discardOnZeroedWriteSameDisabled {
return nil
}
err := cc.conn.SetConfigOption("rbd_discard_on_zeroed_write_same", "false")
if err != nil {
return err
}
cc.discardOnZeroedWriteSameDisabled = true
return nil
}