diff --git a/docs/rbd/deploy.md b/docs/rbd/deploy.md index f27f32df0..fd5bb3715 100644 --- a/docs/rbd/deploy.md +++ b/docs/rbd/deploy.md @@ -73,6 +73,15 @@ make image-cephcsi | `stripeUnit` | no | stripe unit in bytes | | `stripeCount` | no | objects to stripe over before looping | | `objectSize` | no | object size in bytes | +| `BaseReadIops` | no | the base limit of read operations per second | +| `BaseWriteIops` | no | the base limit of write operations per second | +| `BaseReadBytesPerSecond` | no | the base limit of read bytes per second | +| `BaseWriteBytesPerSecond` | no | the base limit of write bytes per second | +| `ReadIopsPerGB` | no | the limit of read operations per GiB | +| `WriteIopsPerGB` | no | the limit of write operations per GiB | +| `ReadBpsPerGB` | no | the limit of read bytes per GiB | +| `WriteBpsPerGB` | no | the limit of write bytes per GiB | +| `BaseVolSizeBytes` | no | the min size of volume what use to calc qos beased on capacity | | `extraDeploy` | no | array of extra objects to deploy with the release | **NOTE:** An accompanying CSI configuration file, needs to be provided to the diff --git a/e2e/rbd.go b/e2e/rbd.go index d47469a8b..adb3b5f34 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -4643,6 +4644,130 @@ var _ = Describe("RBD", func() { validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType) }) + By("validate rbd image qos", func() { + qosParameters := map[string]string{ + "BaseReadIops": "2000", + "BaseWriteIops": "1000", + "BaseReadBytesPerSecond": "209715200", + "BaseWriteBytesPerSecond": "104857600", + } + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + qosParameters, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + defer func() { + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + }() + + // create PVC + pvc, err := loadPVC(pvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + + pvc.Namespace = f.UniqueName + wants := map[string]string{ + "rbd_qos_read_iops_limit": "2000", + "rbd_qos_write_iops_limit": "1000", + "rbd_qos_read_bps_limit": "209715200", + "rbd_qos_write_bps_limit": "104857600", + } + + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate rbd image qos + err = validateQOS(f, pvc, wants) + if err != nil { + framework.Failf("failed to validate qos: %v", err) + } + + // delete pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to delete PVC: %v", err) + } + + qosParameters = map[string]string{ + "BaseReadIops": "2000", + "BaseWriteIops": "1000", + "BaseReadBytesPerSecond": "209715200", + "BaseWriteBytesPerSecond": "104857600", + "ReadIopsPerGB": "20", + "WriteIopsPerGB": "10", + "ReadBpsPerGB": "2097152", + "WriteBpsPerGB": "1048576", + "BaseVolSizeBytes": "21474836480", + } + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + framework.Failf("failed to delete storageclass: %v", err) + } + + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + qosParameters, + deletePolicy) + if err != nil { + framework.Failf("failed to create storageclass: %v", err) + } + + // create PVC + pvc, err = loadPVC(pvcPath) + if err != nil { + framework.Failf("failed to load PVC: %v", err) + } + + pvc.Namespace = f.UniqueName + pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi") + wants = map[string]string{ + "rbd_qos_read_iops_limit": "3600", + "rbd_qos_write_iops_limit": "1800", + "rbd_qos_read_bps_limit": "377487360", + "rbd_qos_write_bps_limit": "188743680", + } + + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to create PVC and application: %v", err) + } + + // validate rbd image qos + err = validateQOS(f, pvc, wants) + if err != nil { + framework.Failf("failed to validate qos: %v", err) + } + + // delete pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + framework.Failf("failed to delete PVC: %v", err) + } + }) + By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() { err := createRBDSnapshotClass(f) if err != nil { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 786660722..87bf007f5 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -1166,3 +1166,28 @@ func validateStripe(f *framework.Framework, return nil } + +func validateQOS(f *framework.Framework, + pvc *v1.PersistentVolumeClaim, + wants map[string]string, +) error { + metadataConfPrefix := "conf_" + + imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) + if err != nil { + return err + } + + rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName) + for k, v := range wants { + qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+k, f) + if err != nil { + return err + } + if qosVal != v { + return fmt.Errorf("%s: %s does not match expected %s", k, qosVal, v) + } + } + + return nil +} diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 9d87b03ef..8e01a3b3b 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -162,6 +162,38 @@ parameters: # stripeCount: <> # (optional) The object size in bytes. # objectSize: <> + + # rbd volume QoS. + # QoS provides settings for rbd volume read/write iops + # and read/write bandwidth. There are 4 base qos parameters + # among them, when users apply for a volume capacity equal + # to or less than BaseVolSizebytes, use base qos limit. + # For the portion of capacity exceeding BaseVolSizebytes, + # QoS will be increased in steps set per GiB. If the step + # size parameter per GiB is not provided, only base QoS limit + # will be used and not associated with capacity size. + # + # note: currently supports rbd-nbd mounter. + # + # For more details + # (optional) the base limit of read operations per second. + # BaseReadIops: <> + # (optional) the base limit of write operations per second. + # BaseWriteIops: <> + # (optional) the base limit of read bytes per second. + # BaseReadBytesPerSecond: <> + # (optional) the base limit of write bytes per second. + # BaseWriteBytesPerSecond: <> + # (optional) the limit of read operations per GiB. + # ReadIopsPerGB: <> + # (optional) the limit of write operations per GiB. + # WriteIopsPerGB: <> + # (optional) the limit of read bytes per GiB. + # ReadBpsPerGB: <> + # (optional) the limit of write bytes per GiB. + # WriteBpsPerGB: <> + # (optional) min size of volume what use to calc qos beased on capacity. + # BaseVolSizeBytes:<> reclaimPolicy: Delete allowVolumeExpansion: true diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 3ec067c3e..e34f00bcb 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest( return nil, status.Error(codes.InvalidArgument, err.Error()) } + // Get QosParameters from SC if qos configuration existing in SC + err = rbdVol.SetQOS(ctx, req.GetParameters()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + err = rbdVol.Connect(cr) if err != nil { log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) @@ -415,7 +421,7 @@ func (cs *ControllerServer) CreateVolume( } }() - err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap) + err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap, req.GetParameters()) if err != nil { if errors.Is(err, ErrFlattenInProgress) { return nil, status.Error(codes.Aborted, err.Error()) @@ -712,6 +718,7 @@ func (cs *ControllerServer) createBackingImage( secrets map[string]string, rbdVol, parentVol *rbdVolume, rbdSnap *rbdSnapshot, + scParams map[string]string, ) error { var err error @@ -766,6 +773,21 @@ func (cs *ControllerServer) createBackingImage( return status.Error(codes.Internal, err.Error()) } + // Apply Qos parameters to rbd image. + err = rbdVol.ApplyQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed apply QOS for rbd image: %v", err) + + return status.Error(codes.Internal, err.Error()) + } + // Save Qos parameters from SC in Image medatate, we will use it while resize volume. + err = rbdVol.SaveQOS(ctx, scParams) + if err != nil { + log.DebugLog(ctx, "failed save QOS for rbd image: %v", err) + + return status.Error(codes.Internal, err.Error()) + } + return nil } @@ -1605,6 +1627,13 @@ func (cs *ControllerServer) ControllerExpandVolume( if err != nil { log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err) + return nil, status.Error(codes.Internal, err.Error()) + } + // adjust rbd qos after resize volume. + err = rbdVol.AdjustQOS(ctx) + if err != nil { + log.DebugLog(ctx, "failed adjust QOS for rbd image") + return nil, status.Error(codes.Internal, err.Error()) } } diff --git a/internal/rbd/qos.go b/internal/rbd/qos.go new file mode 100644 index 000000000..e831a5706 --- /dev/null +++ b/internal/rbd/qos.go @@ -0,0 +1,281 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "errors" + "strconv" + + "github.com/ceph/ceph-csi/internal/util/log" + + librbd "github.com/ceph/go-ceph/rbd" +) + +const ( + // Qos parameters name of StorageClass. + baseReadIops = "BaseReadIops" + baseWriteIops = "BaseWriteIops" + baseReadBytesPerSecond = "BaseReadBytesPerSecond" + baseWriteBytesPerSecond = "BaseWriteBytesPerSecond" + readIopsPerGB = "ReadIopsPerGB" + writeIopsPerGB = "WriteIopsPerGB" + readBpsPerGB = "ReadBpsPerGB" + writeBpsPerGB = "WriteBpsPerGB" + baseVolSizeBytes = "BaseVolSizeBytes" + + // Qos type name of rbd image. + readIopsLimit = "rbd_qos_read_iops_limit" + writeIopsLimit = "rbd_qos_write_iops_limit" + readBpsLimit = "rbd_qos_read_bps_limit" + writeBpsLimit = "rbd_qos_write_bps_limit" + metadataConfPrefix = "conf_" + + // The params use to calc qos based on capacity. + baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit" + baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit" + baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit" + baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit" + readIopsPerGBLimit = "rbd_read_iops_per_gb_limit" + writeIopsPerGBLimit = "rbd_write_iops_per_gb_limit" + readBpsPerGBLimit = "rbd_read_bps_per_gb_limit" + writeBpsPerGBLimit = "rbd_write_bps_per_gb_limit" + baseQosVolSize = "rbd_base_qos_vol_size" +) + +type qosSpec struct { + baseLimitType string + baseLimit string + perGBLimitType string + perGBLimit string + provide bool +} + +func parseQosParams( + scParams map[string]string, +) map[string]*qosSpec { + rbdQosParameters := map[string]*qosSpec{ + baseReadIops: {readIopsLimit, "", readIopsPerGB, "", false}, + baseWriteIops: {writeIopsLimit, "", writeIopsPerGB, "", false}, + baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGB, "", false}, + baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGB, "", false}, + } + for k, v := range scParams { + if qos, ok := rbdQosParameters[k]; ok && v != "" { + qos.baseLimit = v + qos.provide = true + for _k, _v := range scParams { + if _k == qos.perGBLimitType && _v != "" { + qos.perGBLimit = _v + } + } + } + } + + return rbdQosParameters +} + +func (rv *rbdVolume) SetQOS( + ctx context.Context, + scParams map[string]string, +) error { + rv.BaseVolSize = "" + if v, ok := scParams[baseVolSizeBytes]; ok && v != "" { + rv.BaseVolSize = v + } + + rbdQosParameters := parseQosParams(scParams) + for _, qos := range rbdQosParameters { + if qos.provide { + err := calcQosBasedOnCapacity(ctx, rv, *qos) + if err != nil { + return err + } + } + } + + return nil +} + +func (rv *rbdVolume) ApplyQOS( + ctx context.Context, +) error { + for k, v := range rv.QosParameters { + err := rv.SetMetadata(metadataConfPrefix+k, v) + if err != nil { + log.ErrorLog(ctx, "failed to set rbd qos, %s: %s. %v", k, v, err) + + return err + } + } + + return nil +} + +func calcQosBasedOnCapacity( + ctx context.Context, + rbdVol *rbdVolume, + qos qosSpec, +) error { + if rbdVol.QosParameters == nil { + rbdVol.QosParameters = make(map[string]string) + } + + // Don't set qos if base qos limit empty. + if qos.baseLimit == "" { + return nil + } + baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.baseLimitType, qos.baseLimit, err) + + return err + } + + // if provide qosPerGB and baseVolSize, we will set qos based on capacity, + // otherwise, we only set base qos limit. + if qos.perGBLimit != "" && rbdVol.BaseVolSize != "" { + perGBLimit, err := strconv.ParseInt(qos.perGBLimit, 10, 64) + if err != nil { + log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.perGBLimitType, qos.perGBLimit, err) + + return err + } + + baseVolSize, err := strconv.ParseInt(rbdVol.BaseVolSize, 10, 64) + if err != nil { + log.ErrorLog(ctx, "failed to parse %s: %s. %v", baseVolSizeBytes, rbdVol.BaseVolSize, err) + + return err + } + + if rbdVol.VolSize <= baseVolSize { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } else { + capacityQos := (rbdVol.VolSize - baseVolSize) / int64(oneGB) * perGBLimit + finalQosLimit := baseLimit + capacityQos + rbdVol.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10) + } + } else { + rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit + } + + return nil +} + +func (rv *rbdVolume) SaveQOS( + ctx context.Context, + scParams map[string]string, +) error { + needSaveQosParameters := map[string]string{ + baseReadIops: baseQosReadIopsLimit, + baseWriteIops: baseQosWriteIopsLimit, + baseReadBytesPerSecond: baseQosReadBpsLimit, + baseWriteBytesPerSecond: baseQosWriteBpsLimit, + readIopsPerGB: readIopsPerGBLimit, + writeIopsPerGB: writeIopsPerGBLimit, + readBpsPerGB: readBpsPerGBLimit, + writeBpsPerGB: writeBpsPerGBLimit, + baseVolSizeBytes: baseQosVolSize, + } + for k, v := range scParams { + if param, ok := needSaveQosParameters[k]; ok { + if v != "" { + err := rv.SetMetadata(param, v) + if err != nil { + log.ErrorLog(ctx, "failed to save qos. %s: %s, %v", k, v, err) + + return err + } + } + } + } + + return nil +} + +func getRbdImageQOS( + ctx context.Context, + rbdVol *rbdVolume, +) (map[string]qosSpec, error) { + QosParams := map[string]struct { + rbdQosType string + rbdQosPerGBType string + }{ + baseQosReadIopsLimit: {readIopsLimit, readIopsPerGBLimit}, + baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGBLimit}, + baseQosReadBpsLimit: {readBpsLimit, readBpsPerGBLimit}, + baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGBLimit}, + } + rbdQosParameters := make(map[string]qosSpec) + for k, param := range QosParams { + baseLimit, err := rbdVol.GetMetadata(k) + if errors.Is(err, librbd.ErrNotFound) { + // if base qos dose not exist, skipping. + continue + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %s. %v", k, err) + + return nil, err + } + perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType) + if errors.Is(err, librbd.ErrNotFound) { + // rbdQosPerGBType does not exist, set it empty. + perGBLimit = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %s. %v", param.rbdQosPerGBType, err) + + return nil, err + } + rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true} + } + baseVolSize, err := rbdVol.GetMetadata(baseQosVolSize) + if errors.Is(err, librbd.ErrNotFound) { + // rbdBaseQosVolSize does not exist, set it empty. + baseVolSize = "" + } else if err != nil { + log.ErrorLog(ctx, "failed to get metadata: %s. %v", baseQosVolSize, err) + + return nil, err + } + rbdVol.BaseVolSize = baseVolSize + + return rbdQosParameters, nil +} + +func (rv *rbdVolume) AdjustQOS( + ctx context.Context, +) error { + rbdQosParameters, err := getRbdImageQOS(ctx, rv) + if err != nil { + log.ErrorLog(ctx, "get rbd image qos failed") + + return err + } + for _, param := range rbdQosParameters { + err = calcQosBasedOnCapacity(ctx, rv, param) + if err != nil { + return err + } + } + err = rv.ApplyQOS(ctx) + if err != nil { + return err + } + + return nil +} diff --git a/internal/rbd/qos_test.go b/internal/rbd/qos_test.go new file mode 100644 index 000000000..6dd21fedc --- /dev/null +++ b/internal/rbd/qos_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2024 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "testing" +) + +func checkQOS( + t *testing.T, + target map[string]string, + wants map[string]string, +) { + t.Helper() + + for k, v := range wants { + if r, ok := target[k]; ok { + if v != r { + t.Errorf("SetQOS: %s: %s, want %s", k, target[k], v) + } + } else { + t.Errorf("SetQOS: missing qos %s", k) + } + } +} + +func TestSetQOS(t *testing.T) { + t.Parallel() + ctx := context.TODO() + + tests := map[string]string{ + baseReadIops: "2000", + baseWriteIops: "1000", + } + wants := map[string]string{ + readIopsLimit: "2000", + writeIopsLimit: "1000", + } + rv := rbdVolume{} + rv.VolSize = int64(oneGB) + err := rv.SetQOS(ctx, tests) + if err != nil { + t.Errorf("SetQOS failed: %v", err) + } + checkQOS(t, rv.QosParameters, wants) + + tests = map[string]string{ + baseReadIops: "2000", + baseWriteIops: "1000", + baseReadBytesPerSecond: "209715200", + baseWriteBytesPerSecond: "104857600", + } + wants = map[string]string{ + readIopsLimit: "2000", + writeIopsLimit: "1000", + readBpsLimit: "209715200", + writeBpsLimit: "104857600", + } + rv = rbdVolume{} + rv.VolSize = int64(oneGB) + err = rv.SetQOS(ctx, tests) + if err != nil { + t.Errorf("SetQOS failed: %v", err) + } + checkQOS(t, rv.QosParameters, wants) + + tests = map[string]string{ + baseReadIops: "2000", + baseWriteIops: "1000", + baseReadBytesPerSecond: "209715200", + baseWriteBytesPerSecond: "104857600", + readIopsPerGB: "20", + writeIopsPerGB: "10", + readBpsPerGB: "2097152", + writeBpsPerGB: "1048576", + baseVolSizeBytes: "21474836480", + } + wants = map[string]string{ + readIopsLimit: "2000", + writeIopsLimit: "1000", + readBpsLimit: "209715200", + writeBpsLimit: "104857600", + } + rv = rbdVolume{} + rv.VolSize = int64(oneGB) * 20 + err = rv.SetQOS(ctx, tests) + if err != nil { + t.Errorf("SetQOS failed: %v", err) + } + checkQOS(t, rv.QosParameters, wants) + + wants = map[string]string{ + readIopsLimit: "3600", + writeIopsLimit: "1800", + readBpsLimit: "377487360", + writeBpsLimit: "188743680", + } + rv = rbdVolume{} + rv.VolSize = int64(oneGB) * 100 + err = rv.SetQOS(ctx, tests) + if err != nil { + t.Errorf("SetQOS failed: %v", err) + } + checkQOS(t, rv.QosParameters, wants) +} diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index fbae7f418..822eb53be 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -148,6 +148,12 @@ type rbdImage struct { EnableMetadata bool // ParentInTrash indicates the parent image is in trash. ParentInTrash bool + + // RBD QoS configuration + QosParameters map[string]string + + // the min size of volume what use to calc qos beased on capacity. + BaseVolSize string } // check that rbdVolume implements the types.Volume interface.