mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-18 11:00:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: add exclusive-lock and journaling image features for rbd image

Browse Source 
Current rbd plugin only supports the layering feature
for rbd image. Add exclusive-lock and journaling image
features for the rbd.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
Signed-off-by: woohhan <woohyung_han@tmax.co.kr>
This commit is contained in:
Madhu Rajanna 2021-03-11 11:58:48 +05:30 committed by mergify[bot]
parent 2f6fca0862
commit d8f7b38d3d
5 changed files with 170 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
docs/deploy-rbd.md
View File

@ -48,22 +48,23 @@ make image-cephcsi
**Available volume parameters:** **Available volume parameters:**
| Parameter | Required | Description | | Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | --------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | | `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `pool` | yes | Ceph pool into which the RBD image shall be created | | `pool` | yes | Ceph pool into which the RBD image shall be created |
| `dataPool` | no | Ceph pool used for the data of the RBD images. | | `dataPool` | no | Ceph pool used for the data of the RBD images. |
| `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). | | `volumeNamePrefix` | no | Prefix to use for naming RBD images (defaults to `csi-vol-`). |
| `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). | | `snapshotNamePrefix` | no | Prefix to use for naming RBD snapshot images (defaults to `csi-snap-`). |
| `imageFeatures` | yes | RBD image features. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features) which is why they are unsupported here. deep-flatten is added for cloned images. | | `imageFeatures` | yes | RBD image features. CSI RBD currently supports `layering`, `journaling`, `exclusive-lock` features. If `journaling` is enabled, must enable `exclusive-lock` too. See [man pages](http://docs.ceph.com/docs/master/man/8/rbd/#cmdoption-rbd-image-feature) Note that the required support for [object-map and fast-diff were added in 5.3 and journaling does not have KRBD support yet](https://docs.ceph.com/en/latest/rbd/rbd-config-ref/#image-features). deep-flatten is added for cloned images. |
| `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | | |
| `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. | | `mapOptions` | no | Map options to use when mapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `unmapOptions` | no | Unmap options to use when unmapping rbd image. See [krbd](https://docs.ceph.com/docs/master/man/8/rbd/#kernel-rbd-krbd-options) and [nbd](https://docs.ceph.com/docs/master/man/8/rbd-nbd/#options) options. |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | | `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects |
| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** | | `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images |
| `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases | | `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on PVC and `"false"` to disable it. **Do not change for existing storageclasses** |
| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it | | `encryptionKMSID` | no | required if encryption is enabled and a kms is used to store passphrases |
| `thickProvision` | no | if set to `"true"`, newly created RBD images will be completely allocated by writing zeros to it |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the **NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration) running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)

12
e2e/rbd.go
View File

@ -467,6 +467,18 @@ var _ = Describe("RBD", func() {
} }
}) })
// TODO: enable this test when we support rbd-nbd mounter in E2E.
// nbd module should be present on the host machine to run use the
// rbd-nbd mounter.
// By("create a PVC and Bind it to an app with journaling/exclusive-lock image-features and rbd-nbd mounter", func() {
// deleteResource(rbdExamplePath + "storageclass.yaml")
// createRBDStorageClass(f.ClientSet, f, nil, map[string]string{"imageFeatures": "layering,journaling,exclusive-lock", "mounter": "rbd-nbd"})
// validatePVCAndAppBinding(pvcPath, appPath, f)
// deleteResource(rbdExamplePath + "storageclass.yaml")
// createRBDStorageClass(f.ClientSet, f, nil, make(map[string]string))
// })
By("create a PVC clone and bind it to an app", func() { By("create a PVC clone and bind it to an app", func() {
// snapshot beta is only supported from v1.17+ // snapshot beta is only supported from v1.17+
if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { if k8sVersionGreaterEquals(f.ClientSet, 1, 17) {

4
examples/rbd/storageclass.yaml
View File

@ -33,7 +33,9 @@ parameters:
# creation (thin provisioning is the default). # creation (thin provisioning is the default).
thickProvision: "false" thickProvision: "false"
# (required) RBD image features, CSI creates image with image-format 2 # (required) RBD image features, CSI creates image with image-format 2
# CSI RBD currently supports only `layering` feature. # CSI RBD currently supports `layering`, `journaling`, `exclusive-lock`
# features. If `journaling` is enabled, must enable `exclusive-lock` too.
# imageFeatures: layering,journaling,exclusive-lock
imageFeatures: layering imageFeatures: layering
# (optional) mapOptions is a comma-separated list of map options. # (optional) mapOptions is a comma-separated list of map options.

75
internal/rbd/rbd_util.go
View File

@ -46,6 +46,7 @@ const (
rbdImageWatcherFactor = 1.4 rbdImageWatcherFactor = 1.4
rbdImageWatcherSteps = 10 rbdImageWatcherSteps = 10
rbdDefaultMounter = "rbd" rbdDefaultMounter = "rbd"
rbdNbdMounter = "rbd-nbd"
// Output strings returned during invocation of "ceph rbd task add remove <imagespec>" when // Output strings returned during invocation of "ceph rbd task add remove <imagespec>" when
// command is not supported by ceph manager. Used to check errors and recover when the command // command is not supported by ceph manager. Used to check errors and recover when the command
@ -142,8 +143,27 @@ type rbdSnapshot struct {
SizeBytes int64 SizeBytes int64
} }
// imageFeature represents required image features and value.
type imageFeature struct {
// needRbdNbd indicates whether this image feature requires an rbd-nbd mounter
needRbdNbd bool
// dependsOn is the image features required for this imageFeature
dependsOn []string
}
var ( var (
supportedFeatures = sets.NewString(librbd.FeatureNameLayering) supportedFeatures = map[string]imageFeature{
librbd.FeatureNameLayering: {
needRbdNbd: false,
},
librbd.FeatureNameExclusiveLock: {
needRbdNbd: true,
},
librbd.FeatureNameJournaling: {
needRbdNbd: true,
dependsOn: []string{librbd.FeatureNameExclusiveLock},
},
}
) )
// Connect an rbdVolume to the Ceph cluster. // Connect an rbdVolume to the Ceph cluster.
@ -889,28 +909,18 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st
if err != nil { if err != nil {
return nil, err return nil, err
} }
// if no image features is provided, it results in empty string if rbdVol.Mounter, ok = volOptions["mounter"]; !ok {
// which disable all RBD image features as we expected
imageFeatures, found := volOptions["imageFeatures"]
if found {
arr := strings.Split(imageFeatures, ",")
for _, f := range arr {
if !supportedFeatures.Has(f) {
return nil, fmt.Errorf("invalid feature %q for volume csi-rbdplugin, supported"+
" features are: %v", f, supportedFeatures)
}
}
rbdVol.imageFeatureSet = librbd.FeatureSetFromNames(arr)
}
util.ExtendedLog(ctx, "setting disableInUseChecks on rbd volume to: %v", disableInUseChecks)
rbdVol.DisableInUseChecks = disableInUseChecks
rbdVol.Mounter, ok = volOptions["mounter"]
if !ok {
rbdVol.Mounter = rbdDefaultMounter rbdVol.Mounter = rbdDefaultMounter
} }
// if no image features is provided, it results in empty string
// which disable all RBD image features as we expected
if err = rbdVol.validateImageFeatures(volOptions["imageFeatures"]); err != nil {
util.ErrorLog(ctx, "failed to validate image features %v", err)
return nil, err
}
util.ExtendedLog(ctx, "setting disableInUseChecks: %t image features: %v mounter: %s", disableInUseChecks, rbdVol.imageFeatureSet.Names(), rbdVol.Mounter)
rbdVol.DisableInUseChecks = disableInUseChecks
err = rbdVol.initKMS(ctx, volOptions, credentials) err = rbdVol.initKMS(ctx, volOptions, credentials)
if err != nil { if err != nil {
@ -920,6 +930,29 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st
return rbdVol, nil return rbdVol, nil
} }
func (rv *rbdVolume) validateImageFeatures(imageFeatures string) error {
arr := strings.Split(imageFeatures, ",")
featureSet := sets.NewString(arr...)
for _, f := range arr {
sf, found := supportedFeatures[f]
if !found {
return fmt.Errorf("invalid feature %s", f)
}
for _, r := range sf.dependsOn {
if !featureSet.Has(r) {
return fmt.Errorf("feature %s requires %s to be set", f, r)
}
}
if sf.needRbdNbd && rv.Mounter != rbdNbdMounter {
return fmt.Errorf("feature %s requires rbd-nbd for mounter", f)
}
}
rv.imageFeatureSet = librbd.FeatureSetFromNames(arr)
return nil
}
func genSnapFromOptions(ctx context.Context, rbdVol *rbdVolume, snapOptions map[string]string) (*rbdSnapshot, error) { func genSnapFromOptions(ctx context.Context, rbdVol *rbdVolume, snapOptions map[string]string) (*rbdSnapshot, error) {
var err error var err error

84
internal/rbd/rbd_util_test.go
View File

@ -21,6 +21,7 @@ import (
"testing" "testing"
librbd "github.com/ceph/go-ceph/rbd" librbd "github.com/ceph/go-ceph/rbd"
"github.com/stretchr/testify/assert"
) )
func TestHasSnapshotFeature(t *testing.T) { func TestHasSnapshotFeature(t *testing.T) {
@ -42,3 +43,86 @@ func TestHasSnapshotFeature(t *testing.T) {
} }
} }
} }
func TestValidateImageFeatures(t *testing.T) {
tests := []struct {
imageFeatures string
rbdVol *rbdVolume
isErr bool
errMsg string
}{
{
"layering",
&rbdVolume{
Mounter: rbdDefaultMounter,
},
false,
"",
},
{
"layering",
&rbdVolume{
Mounter: rbdNbdMounter,
},
false,
"",
},
{
"layering,exclusive-lock,journaling",
&rbdVolume{
Mounter: rbdNbdMounter,
},
false,
"",
},
{
"layering,journaling",
&rbdVolume{
Mounter: rbdNbdMounter,
},
true,
"feature journaling requires exclusive-lock to be set",
},
{
"layering,exclusive-lock,journaling",
&rbdVolume{
Mounter: rbdDefaultMounter,
},
true,
"feature exclusive-lock requires rbd-nbd for mounter",
},
{
"layering,exclusive-lock,journaling",
&rbdVolume{
Mounter: rbdDefaultMounter,
},
true,
"feature exclusive-lock requires rbd-nbd for mounter",
},
{
"layering,exclusive-loc,journaling",
&rbdVolume{
Mounter: rbdNbdMounter,
},
true,
"invalid feature exclusive-loc",
},
{
"ayering",
&rbdVolume{
Mounter: rbdDefaultMounter,
},
true,
"invalid feature ayering",
},
}
for _, test := range tests {
err := test.rbdVol.validateImageFeatures(test.imageFeatures)
if test.isErr {
assert.EqualError(t, err, test.errMsg)
continue
}
assert.Nil(t, err)
}
}