mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-24 14:00:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: support QoS based on capacity for rbd volume

Browse Source 
1. QoS provides settings for rbd volume read/write iops
   and read/write bandwidth.
2. All QoS parameters are placed in the SC,
   send QoS parameters from SC to Cephcsi through PVC create request.
3. We need provide QoS parameters in the SC as below:
   - BaseReadIops
   - BaseWriteIops
   - BaseReadBytesPerSecond
   - BaseWriteBytesPerSecond
   - ReadIopsPerGB
   - WriteIopsPerGB
   - ReadBpsPerGB
   - WriteBpsPerGB
   - BaseVolSizeBytes
   There are 4 base qos parameters among them, when users apply for
   a volume capacity equal to or less than BaseVolSizebytes, use base
   qos limit. For the portion of capacity exceeding BaseVolSizebytes,
   QoS will be increased in steps set per GB. If the step size parameter
   per GB is not provided, only base QoS limit will be used and not associated
   with capacity size.
4. If PVC has resize request, adjust the QoS limit
   according to the QoS parameters after resizing.

Signed-off-by: Yite Gu <guyite@bytedance.com>
This commit is contained in:
Yite Gu 2024-12-12 17:26:25 +08:00
parent 3c63fea23a
commit 4774a82e88
8 changed files with 628 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/rbd/deploy.md
View File

@ -73,6 +73,15 @@ make image-cephcsi
| `stripeUnit` | no | stripe unit in bytes |
| `stripeCount` | no | objects to stripe over before looping |
| `objectSize` | no | object size in bytes |
| `BaseReadIops` | no | the base limit of read operations per second |
| `BaseWriteIops` | no | the base limit of write operations per second |
| `BaseReadBytesPerSecond` | no | the base limit of read bytes per second |
| `BaseWriteBytesPerSecond` | no | the base limit of write bytes per second |
| `ReadIopsPerGB` | no | the limit of read operations per GiB |
| `WriteIopsPerGB` | no | the limit of write operations per GiB |
| `ReadBpsPerGB` | no | the limit of read bytes per GiB |
| `WriteBpsPerGB` | no | the limit of write bytes per GiB |
| `BaseVolSizeBytes` | no | the min size of volume what use to calc qos beased on capacity |
| `extraDeploy` | no | array of extra objects to deploy with the release |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the

125
e2e/rbd.go
View File

@ -27,6 +27,7 @@ import (
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
@ -4643,6 +4644,130 @@ var _ = Describe("RBD", func() {
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})
By("validate rbd image qos", func() {
qosParameters := map[string]string{
"BaseReadIops": "2000",
"BaseWriteIops": "1000",
"BaseReadBytesPerSecond": "209715200",
"BaseWriteBytesPerSecond": "104857600",
}
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
}()
// create PVC
pvc, err := loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
wants := map[string]string{
"rbd_qos_read_iops_limit": "2000",
"rbd_qos_write_iops_limit": "1000",
"rbd_qos_read_bps_limit": "209715200",
"rbd_qos_write_bps_limit": "104857600",
}
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate rbd image qos
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// delete pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
qosParameters = map[string]string{
"BaseReadIops": "2000",
"BaseWriteIops": "1000",
"BaseReadBytesPerSecond": "209715200",
"BaseWriteBytesPerSecond": "104857600",
"ReadIopsPerGB": "20",
"WriteIopsPerGB": "10",
"ReadBpsPerGB": "2097152",
"WriteBpsPerGB": "1048576",
"BaseVolSizeBytes": "21474836480",
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
// create PVC
pvc, err = loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
wants = map[string]string{
"rbd_qos_read_iops_limit": "3600",
"rbd_qos_write_iops_limit": "1800",
"rbd_qos_read_bps_limit": "377487360",
"rbd_qos_write_bps_limit": "188743680",
}
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate rbd image qos
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// delete pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
})
By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() {
err := createRBDSnapshotClass(f)
if err != nil {

25
e2e/rbd_helper.go
View File

@ -1166,3 +1166,28 @@ func validateStripe(f *framework.Framework,
return nil
}
func validateQOS(f *framework.Framework,
pvc *v1.PersistentVolumeClaim,
wants map[string]string,
) error {
metadataConfPrefix := "conf_"
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
for k, v := range wants {
qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+k, f)
if err != nil {
return err
}
if qosVal != v {
return fmt.Errorf("%s: %s does not match expected %s", k, qosVal, v)
}
}
return nil
}

32
examples/rbd/storageclass.yaml
View File

@ -162,6 +162,38 @@ parameters:
# stripeCount: <>
# (optional) The object size in bytes.
# objectSize: <>
# rbd volume QoS.
# QoS provides settings for rbd volume read/write iops
# and read/write bandwidth. There are 4 base qos parameters
# among them, when users apply for a volume capacity equal
# to or less than BaseVolSizebytes, use base qos limit.
# For the portion of capacity exceeding BaseVolSizebytes,
# QoS will be increased in steps set per GiB. If the step
# size parameter per GiB is not provided, only base QoS limit
# will be used and not associated with capacity size.
#
# note: currently supports rbd-nbd mounter.
#
# For more details
# (optional) the base limit of read operations per second.
# BaseReadIops: <>
# (optional) the base limit of write operations per second.
# BaseWriteIops: <>
# (optional) the base limit of read bytes per second.
# BaseReadBytesPerSecond: <>
# (optional) the base limit of write bytes per second.
# BaseWriteBytesPerSecond: <>
# (optional) the limit of read operations per GiB.
# ReadIopsPerGB: <>
# (optional) the limit of write operations per GiB.
# WriteIopsPerGB: <>
# (optional) the limit of read bytes per GiB.
# ReadBpsPerGB: <>
# (optional) the limit of write bytes per GiB.
# WriteBpsPerGB: <>
# (optional) min size of volume what use to calc qos beased on capacity.
# BaseVolSizeBytes:<>
reclaimPolicy: Delete
allowVolumeExpansion: true

31
internal/rbd/controllerserver.go
View File

@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest(
return nil, status.Error(codes.InvalidArgument, err.Error())
}
// Get QosParameters from SC if qos configuration existing in SC
err = rbdVol.SetQOS(ctx, req.GetParameters())
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
err = rbdVol.Connect(cr)
if err != nil {
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
@ -415,7 +421,7 @@ func (cs *ControllerServer) CreateVolume(
}
}()
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap)
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap, req.GetParameters())
if err != nil {
if errors.Is(err, ErrFlattenInProgress) {
return nil, status.Error(codes.Aborted, err.Error())
@ -712,6 +718,7 @@ func (cs *ControllerServer) createBackingImage(
secrets map[string]string,
rbdVol, parentVol *rbdVolume,
rbdSnap *rbdSnapshot,
scParams map[string]string,
) error {
var err error
@ -766,6 +773,21 @@ func (cs *ControllerServer) createBackingImage(
return status.Error(codes.Internal, err.Error())
}
// Apply Qos parameters to rbd image.
err = rbdVol.ApplyQOS(ctx)
if err != nil {
log.DebugLog(ctx, "failed apply QOS for rbd image: %v", err)
return status.Error(codes.Internal, err.Error())
}
// Save Qos parameters from SC in Image medatate, we will use it while resize volume.
err = rbdVol.SaveQOS(ctx, scParams)
if err != nil {
log.DebugLog(ctx, "failed save QOS for rbd image: %v", err)
return status.Error(codes.Internal, err.Error())
}
return nil
}
@ -1605,6 +1627,13 @@ func (cs *ControllerServer) ControllerExpandVolume(
if err != nil {
log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err)
return nil, status.Error(codes.Internal, err.Error())
}
// adjust rbd qos after resize volume.
err = rbdVol.AdjustQOS(ctx)
if err != nil {
log.DebugLog(ctx, "failed adjust QOS for rbd image")
return nil, status.Error(codes.Internal, err.Error())
}
}

281
internal/rbd/qos.go Normal file
View File

@ -0,0 +1,281 @@
/*
Copyright 2024 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"context"
"errors"
"strconv"
"github.com/ceph/ceph-csi/internal/util/log"
librbd "github.com/ceph/go-ceph/rbd"
)
const (
// Qos parameters name of StorageClass.
baseReadIops = "BaseReadIops"
baseWriteIops = "BaseWriteIops"
baseReadBytesPerSecond = "BaseReadBytesPerSecond"
baseWriteBytesPerSecond = "BaseWriteBytesPerSecond"
readIopsPerGB = "ReadIopsPerGB"
writeIopsPerGB = "WriteIopsPerGB"
readBpsPerGB = "ReadBpsPerGB"
writeBpsPerGB = "WriteBpsPerGB"
baseVolSizeBytes = "BaseVolSizeBytes"
// Qos type name of rbd image.
readIopsLimit = "rbd_qos_read_iops_limit"
writeIopsLimit = "rbd_qos_write_iops_limit"
readBpsLimit = "rbd_qos_read_bps_limit"
writeBpsLimit = "rbd_qos_write_bps_limit"
metadataConfPrefix = "conf_"
// The params use to calc qos based on capacity.
baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit"
baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit"
baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit"
baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit"
readIopsPerGBLimit = "rbd_read_iops_per_gb_limit"
writeIopsPerGBLimit = "rbd_write_iops_per_gb_limit"
readBpsPerGBLimit = "rbd_read_bps_per_gb_limit"
writeBpsPerGBLimit = "rbd_write_bps_per_gb_limit"
baseQosVolSize = "rbd_base_qos_vol_size"
)
type qosSpec struct {
baseLimitType string
baseLimit string
perGBLimitType string
perGBLimit string
provide bool
}
func parseQosParams(
scParams map[string]string,
) map[string]*qosSpec {
rbdQosParameters := map[string]*qosSpec{
baseReadIops: {readIopsLimit, "", readIopsPerGB, "", false},
baseWriteIops: {writeIopsLimit, "", writeIopsPerGB, "", false},
baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGB, "", false},
baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGB, "", false},
}
for k, v := range scParams {
if qos, ok := rbdQosParameters[k]; ok && v != "" {
qos.baseLimit = v
qos.provide = true
for _k, _v := range scParams {
if _k == qos.perGBLimitType && _v != "" {
qos.perGBLimit = _v
}
}
}
}
return rbdQosParameters
}
func (rv *rbdVolume) SetQOS(
ctx context.Context,
scParams map[string]string,
) error {
rv.BaseVolSize = ""
if v, ok := scParams[baseVolSizeBytes]; ok && v != "" {
rv.BaseVolSize = v
}
rbdQosParameters := parseQosParams(scParams)
for _, qos := range rbdQosParameters {
if qos.provide {
err := calcQosBasedOnCapacity(ctx, rv, *qos)
if err != nil {
return err
}
}
}
return nil
}
func (rv *rbdVolume) ApplyQOS(
ctx context.Context,
) error {
for k, v := range rv.QosParameters {
err := rv.SetMetadata(metadataConfPrefix+k, v)
if err != nil {
log.ErrorLog(ctx, "failed to set rbd qos, %s: %s. %v", k, v, err)
return err
}
}
return nil
}
func calcQosBasedOnCapacity(
ctx context.Context,
rbdVol *rbdVolume,
qos qosSpec,
) error {
if rbdVol.QosParameters == nil {
rbdVol.QosParameters = make(map[string]string)
}
// Don't set qos if base qos limit empty.
if qos.baseLimit == "" {
return nil
}
baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64)
if err != nil {
log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.baseLimitType, qos.baseLimit, err)
return err
}
// if provide qosPerGB and baseVolSize, we will set qos based on capacity,
// otherwise, we only set base qos limit.
if qos.perGBLimit != "" && rbdVol.BaseVolSize != "" {
perGBLimit, err := strconv.ParseInt(qos.perGBLimit, 10, 64)
if err != nil {
log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.perGBLimitType, qos.perGBLimit, err)
return err
}
baseVolSize, err := strconv.ParseInt(rbdVol.BaseVolSize, 10, 64)
if err != nil {
log.ErrorLog(ctx, "failed to parse %s: %s. %v", baseVolSizeBytes, rbdVol.BaseVolSize, err)
return err
}
if rbdVol.VolSize <= baseVolSize {
rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit
} else {
capacityQos := (rbdVol.VolSize - baseVolSize) / int64(oneGB) * perGBLimit
finalQosLimit := baseLimit + capacityQos
rbdVol.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10)
}
} else {
rbdVol.QosParameters[qos.baseLimitType] = qos.baseLimit
}
return nil
}
func (rv *rbdVolume) SaveQOS(
ctx context.Context,
scParams map[string]string,
) error {
needSaveQosParameters := map[string]string{
baseReadIops: baseQosReadIopsLimit,
baseWriteIops: baseQosWriteIopsLimit,
baseReadBytesPerSecond: baseQosReadBpsLimit,
baseWriteBytesPerSecond: baseQosWriteBpsLimit,
readIopsPerGB: readIopsPerGBLimit,
writeIopsPerGB: writeIopsPerGBLimit,
readBpsPerGB: readBpsPerGBLimit,
writeBpsPerGB: writeBpsPerGBLimit,
baseVolSizeBytes: baseQosVolSize,
}
for k, v := range scParams {
if param, ok := needSaveQosParameters[k]; ok {
if v != "" {
err := rv.SetMetadata(param, v)
if err != nil {
log.ErrorLog(ctx, "failed to save qos. %s: %s, %v", k, v, err)
return err
}
}
}
}
return nil
}
func getRbdImageQOS(
ctx context.Context,
rbdVol *rbdVolume,
) (map[string]qosSpec, error) {
QosParams := map[string]struct {
rbdQosType string
rbdQosPerGBType string
}{
baseQosReadIopsLimit: {readIopsLimit, readIopsPerGBLimit},
baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGBLimit},
baseQosReadBpsLimit: {readBpsLimit, readBpsPerGBLimit},
baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGBLimit},
}
rbdQosParameters := make(map[string]qosSpec)
for k, param := range QosParams {
baseLimit, err := rbdVol.GetMetadata(k)
if errors.Is(err, librbd.ErrNotFound) {
// if base qos dose not exist, skipping.
continue
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %s. %v", k, err)
return nil, err
}
perGBLimit, err := rbdVol.GetMetadata(param.rbdQosPerGBType)
if errors.Is(err, librbd.ErrNotFound) {
// rbdQosPerGBType does not exist, set it empty.
perGBLimit = ""
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %s. %v", param.rbdQosPerGBType, err)
return nil, err
}
rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGBType, perGBLimit, true}
}
baseVolSize, err := rbdVol.GetMetadata(baseQosVolSize)
if errors.Is(err, librbd.ErrNotFound) {
// rbdBaseQosVolSize does not exist, set it empty.
baseVolSize = ""
} else if err != nil {
log.ErrorLog(ctx, "failed to get metadata: %s. %v", baseQosVolSize, err)
return nil, err
}
rbdVol.BaseVolSize = baseVolSize
return rbdQosParameters, nil
}
func (rv *rbdVolume) AdjustQOS(
ctx context.Context,
) error {
rbdQosParameters, err := getRbdImageQOS(ctx, rv)
if err != nil {
log.ErrorLog(ctx, "get rbd image qos failed")
return err
}
for _, param := range rbdQosParameters {
err = calcQosBasedOnCapacity(ctx, rv, param)
if err != nil {
return err
}
}
err = rv.ApplyQOS(ctx)
if err != nil {
return err
}
return nil
}

120
internal/rbd/qos_test.go Normal file
View File

@ -0,0 +1,120 @@
/*
Copyright 2024 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbd
import (
"context"
"testing"
)
func checkQOS(
t *testing.T,
target map[string]string,
wants map[string]string,
) {
t.Helper()
for k, v := range wants {
if r, ok := target[k]; ok {
if v != r {
t.Errorf("SetQOS: %s: %s, want %s", k, target[k], v)
}
} else {
t.Errorf("SetQOS: missing qos %s", k)
}
}
}
func TestSetQOS(t *testing.T) {
t.Parallel()
ctx := context.TODO()
tests := map[string]string{
baseReadIops: "2000",
baseWriteIops: "1000",
}
wants := map[string]string{
readIopsLimit: "2000",
writeIopsLimit: "1000",
}
rv := rbdVolume{}
rv.VolSize = int64(oneGB)
err := rv.SetQOS(ctx, tests)
if err != nil {
t.Errorf("SetQOS failed: %v", err)
}
checkQOS(t, rv.QosParameters, wants)
tests = map[string]string{
baseReadIops: "2000",
baseWriteIops: "1000",
baseReadBytesPerSecond: "209715200",
baseWriteBytesPerSecond: "104857600",
}
wants = map[string]string{
readIopsLimit: "2000",
writeIopsLimit: "1000",
readBpsLimit: "209715200",
writeBpsLimit: "104857600",
}
rv = rbdVolume{}
rv.VolSize = int64(oneGB)
err = rv.SetQOS(ctx, tests)
if err != nil {
t.Errorf("SetQOS failed: %v", err)
}
checkQOS(t, rv.QosParameters, wants)
tests = map[string]string{
baseReadIops: "2000",
baseWriteIops: "1000",
baseReadBytesPerSecond: "209715200",
baseWriteBytesPerSecond: "104857600",
readIopsPerGB: "20",
writeIopsPerGB: "10",
readBpsPerGB: "2097152",
writeBpsPerGB: "1048576",
baseVolSizeBytes: "21474836480",
}
wants = map[string]string{
readIopsLimit: "2000",
writeIopsLimit: "1000",
readBpsLimit: "209715200",
writeBpsLimit: "104857600",
}
rv = rbdVolume{}
rv.VolSize = int64(oneGB) * 20
err = rv.SetQOS(ctx, tests)
if err != nil {
t.Errorf("SetQOS failed: %v", err)
}
checkQOS(t, rv.QosParameters, wants)
wants = map[string]string{
readIopsLimit: "3600",
writeIopsLimit: "1800",
readBpsLimit: "377487360",
writeBpsLimit: "188743680",
}
rv = rbdVolume{}
rv.VolSize = int64(oneGB) * 100
err = rv.SetQOS(ctx, tests)
if err != nil {
t.Errorf("SetQOS failed: %v", err)
}
checkQOS(t, rv.QosParameters, wants)
}

6
internal/rbd/rbd_util.go
View File

@ -148,6 +148,12 @@ type rbdImage struct {
EnableMetadata bool
// ParentInTrash indicates the parent image is in trash.
ParentInTrash bool
// RBD QoS configuration
QosParameters map[string]string
// the min size of volume what use to calc qos beased on capacity.
BaseVolSize string
}
// check that rbdVolume implements the types.Volume interface.