mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 10:33:35 +00:00
rbd: support QoS based on capacity for rbd volume
1. QoS provides settings for rbd volume read/write iops and read/write bandwidth. 2. All QoS parameters are placed in the SC, send QoS parameters from SC to Cephcsi through PVC create request. 3. We need provide QoS parameters in the SC as below: - BaseReadIops - BaseWriteIops - BaseReadBytesPerSecond - BaseWriteBytesPerSecond - ReadIopsPerGB - WriteIopsPerGB - ReadBpsPerGB - WriteBpsPerGB - BaseVolSizeBytes There are 4 base qos parameters among them, when users apply for a volume capacity equal to or less than BaseVolSizebytes, use base qos limit. For the portion of capacity exceeding BaseVolSizebytes, QoS will be increased in steps set per GB. If the step size parameter per GB is not provided, only base QoS limit will be used and not associated with capacity size. 4. If PVC has resize request, adjust the QoS limit according to the QoS parameters after resizing. Signed-off-by: Yite Gu <guyite@bytedance.com>
This commit is contained in:
Yite Gu
mergify[bot]
@ -176,6 +176,14 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol
|
||||
return err
|
||||
}
|
||||
|
||||
// adjust rbd qos after resize volume.
|
||||
err = rv.AdjustQOS(ctx)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed adjust QOS for rbd image")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -231,6 +231,12 @@ func (cs *ControllerServer) parseVolCreateRequest(
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
// Get QosParameters from SC if qos configuration existing in SC
|
||||
err = rbdVol.SetQOS(ctx, req.GetParameters())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
err = rbdVol.Connect(cr)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err)
|
||||
@ -415,7 +421,7 @@ func (cs *ControllerServer) CreateVolume(
|
||||
}
|
||||
}()
|
||||
|
||||
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap)
|
||||
err = cs.createBackingImage(ctx, cr, req.GetSecrets(), rbdVol, parentVol, rbdSnap, req.GetParameters())
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrFlattenInProgress) {
|
||||
return nil, status.Error(codes.Aborted, err.Error())
|
||||
@ -732,6 +738,7 @@ func (cs *ControllerServer) createBackingImage(
|
||||
secrets map[string]string,
|
||||
rbdVol, parentVol *rbdVolume,
|
||||
rbdSnap *rbdSnapshot,
|
||||
scParams map[string]string,
|
||||
) error {
|
||||
var err error
|
||||
|
||||
@ -786,6 +793,21 @@ func (cs *ControllerServer) createBackingImage(
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
// Apply Qos parameters to rbd image.
|
||||
err = rbdVol.ApplyQOS(ctx)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to apply QOS for rbd image: %s with error: %v", rbdVol, err)
|
||||
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
// Save Qos parameters from SC in Image metadata, we will use it while resize volume.
|
||||
err = rbdVol.SaveQOS(ctx, scParams)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to save QOS for rbd image: %s with error: %v", rbdVol, err)
|
||||
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -1602,6 +1624,13 @@ func (cs *ControllerServer) ControllerExpandVolume(
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err)
|
||||
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
// adjust rbd qos after resize volume.
|
||||
err = rbdVol.AdjustQOS(ctx)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to adjust QOS for rbd image: %s with error: %v", rbdVol, err)
|
||||
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
269
internal/rbd/qos.go
Normal file
269
internal/rbd/qos.go
Normal file
@ -0,0 +1,269 @@
|
||||
/*
|
||||
Copyright 2024 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strconv"
|
||||
|
||||
"github.com/ceph/ceph-csi/internal/util/log"
|
||||
|
||||
librbd "github.com/ceph/go-ceph/rbd"
|
||||
)
|
||||
|
||||
const (
|
||||
// Qos parameters name of StorageClass.
|
||||
baseReadIops = "BaseReadIops"
|
||||
baseWriteIops = "BaseWriteIops"
|
||||
baseReadBytesPerSecond = "BaseReadBytesPerSecond"
|
||||
baseWriteBytesPerSecond = "BaseWriteBytesPerSecond"
|
||||
readIopsPerGiB = "ReadIopsPerGiB"
|
||||
writeIopsPerGiB = "WriteIopsPerGiB"
|
||||
readBpsPerGiB = "ReadBpsPerGiB"
|
||||
writeBpsPerGiB = "WriteBpsPerGiB"
|
||||
baseVolSizeBytes = "BaseVolSizeBytes"
|
||||
|
||||
// Qos type name of rbd image.
|
||||
readIopsLimit = "rbd_qos_read_iops_limit"
|
||||
writeIopsLimit = "rbd_qos_write_iops_limit"
|
||||
readBpsLimit = "rbd_qos_read_bps_limit"
|
||||
writeBpsLimit = "rbd_qos_write_bps_limit"
|
||||
metadataConfPrefix = "conf_"
|
||||
|
||||
// The params use to calc qos based on capacity.
|
||||
baseQosReadIopsLimit = "rbd_base_qos_read_iops_limit"
|
||||
baseQosWriteIopsLimit = "rbd_base_qos_write_iops_limit"
|
||||
baseQosReadBpsLimit = "rbd_base_qos_read_bps_limit"
|
||||
baseQosWriteBpsLimit = "rbd_base_qos_write_bps_limit"
|
||||
readIopsPerGiBLimit = "rbd_read_iops_per_gib_limit"
|
||||
writeIopsPerGiBLimit = "rbd_write_iops_per_gib_limit"
|
||||
readBpsPerGiBLimit = "rbd_read_bps_per_gib_limit"
|
||||
writeBpsPerGiBLimit = "rbd_write_bps_per_gib_limit"
|
||||
baseQosVolSize = "rbd_base_qos_vol_size"
|
||||
)
|
||||
|
||||
type qosSpec struct {
|
||||
baseLimitType string
|
||||
baseLimit string
|
||||
perGiBLimitType string
|
||||
perGiBLimit string
|
||||
present bool
|
||||
}
|
||||
|
||||
func parseQosParams(
|
||||
scParams map[string]string,
|
||||
) map[string]*qosSpec {
|
||||
rbdQosParameters := map[string]*qosSpec{
|
||||
baseReadIops: {readIopsLimit, "", readIopsPerGiB, "", false},
|
||||
baseWriteIops: {writeIopsLimit, "", writeIopsPerGiB, "", false},
|
||||
baseReadBytesPerSecond: {readBpsLimit, "", readBpsPerGiB, "", false},
|
||||
baseWriteBytesPerSecond: {writeBpsLimit, "", writeBpsPerGiB, "", false},
|
||||
}
|
||||
for k, v := range scParams {
|
||||
if qos, ok := rbdQosParameters[k]; ok && v != "" {
|
||||
qos.baseLimit = v
|
||||
qos.present = true
|
||||
if perGiBLimit, ok := scParams[qos.perGiBLimitType]; ok && perGiBLimit != "" {
|
||||
qos.perGiBLimit = perGiBLimit
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rbdQosParameters
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) SetQOS(
|
||||
ctx context.Context,
|
||||
scParams map[string]string,
|
||||
) error {
|
||||
rv.BaseVolSize = ""
|
||||
if v, ok := scParams[baseVolSizeBytes]; ok && v != "" {
|
||||
rv.BaseVolSize = v
|
||||
}
|
||||
|
||||
rbdQosParameters := parseQosParams(scParams)
|
||||
for _, qos := range rbdQosParameters {
|
||||
if qos.present {
|
||||
err := rv.calcQosBasedOnCapacity(ctx, *qos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) ApplyQOS(
|
||||
ctx context.Context,
|
||||
) error {
|
||||
for k, v := range rv.QosParameters {
|
||||
err := rv.SetMetadata(metadataConfPrefix+k, v)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to set rbd qos, %s: %s. %v", k, v, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) calcQosBasedOnCapacity(
|
||||
ctx context.Context,
|
||||
qos qosSpec,
|
||||
) error {
|
||||
if rv.QosParameters == nil {
|
||||
rv.QosParameters = make(map[string]string)
|
||||
}
|
||||
|
||||
// Don't set qos if base qos limit empty.
|
||||
if qos.baseLimit == "" {
|
||||
return nil
|
||||
}
|
||||
baseLimit, err := strconv.ParseInt(qos.baseLimit, 10, 64)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.baseLimitType, qos.baseLimit, err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// if present qosPerGB and baseVolSize, we will set qos based on capacity,
|
||||
// otherwise, we only set base qos limit.
|
||||
if qos.perGiBLimit != "" && rv.BaseVolSize != "" {
|
||||
perGiBLimit, err := strconv.ParseInt(qos.perGiBLimit, 10, 64)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to parse %s: %s. %v", qos.perGiBLimitType, qos.perGiBLimit, err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
baseVolSize, err := strconv.ParseInt(rv.BaseVolSize, 10, 64)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to parse %s: %s. %v", baseVolSizeBytes, rv.BaseVolSize, err)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if rv.RequestedVolSize <= baseVolSize {
|
||||
rv.QosParameters[qos.baseLimitType] = qos.baseLimit
|
||||
} else {
|
||||
capacityQos := (rv.RequestedVolSize - baseVolSize) / int64(oneGB) * perGiBLimit
|
||||
finalQosLimit := baseLimit + capacityQos
|
||||
rv.QosParameters[qos.baseLimitType] = strconv.FormatInt(finalQosLimit, 10)
|
||||
}
|
||||
} else {
|
||||
rv.QosParameters[qos.baseLimitType] = qos.baseLimit
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) SaveQOS(
|
||||
ctx context.Context,
|
||||
scParams map[string]string,
|
||||
) error {
|
||||
needSaveQosParameters := map[string]string{
|
||||
baseReadIops: baseQosReadIopsLimit,
|
||||
baseWriteIops: baseQosWriteIopsLimit,
|
||||
baseReadBytesPerSecond: baseQosReadBpsLimit,
|
||||
baseWriteBytesPerSecond: baseQosWriteBpsLimit,
|
||||
readIopsPerGiB: readIopsPerGiBLimit,
|
||||
writeIopsPerGiB: writeIopsPerGiBLimit,
|
||||
readBpsPerGiB: readBpsPerGiBLimit,
|
||||
writeBpsPerGiB: writeBpsPerGiBLimit,
|
||||
baseVolSizeBytes: baseQosVolSize,
|
||||
}
|
||||
for k, v := range scParams {
|
||||
if param, ok := needSaveQosParameters[k]; ok && v != "" {
|
||||
err := rv.SetMetadata(param, v)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "failed to save qos. %s: %s, %v", k, v, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) getRbdImageQOS(
|
||||
ctx context.Context,
|
||||
) (map[string]qosSpec, error) {
|
||||
QosParams := map[string]struct {
|
||||
rbdQosType string
|
||||
rbdQosPerGiBType string
|
||||
}{
|
||||
baseQosReadIopsLimit: {readIopsLimit, readIopsPerGiBLimit},
|
||||
baseQosWriteIopsLimit: {writeIopsLimit, writeIopsPerGiBLimit},
|
||||
baseQosReadBpsLimit: {readBpsLimit, readBpsPerGiBLimit},
|
||||
baseQosWriteBpsLimit: {writeBpsLimit, writeBpsPerGiBLimit},
|
||||
}
|
||||
rbdQosParameters := make(map[string]qosSpec)
|
||||
for k, param := range QosParams {
|
||||
baseLimit, err := rv.GetMetadata(k)
|
||||
if err != nil && !errors.Is(err, librbd.ErrNotFound) {
|
||||
log.ErrorLog(ctx, "failed to get metadata: %s. %v", k, err)
|
||||
|
||||
return nil, err
|
||||
}
|
||||
if baseLimit == "" {
|
||||
// if base qos dose not exist, skipping.
|
||||
continue
|
||||
}
|
||||
perGiBLimit, err := rv.GetMetadata(param.rbdQosPerGiBType)
|
||||
if err != nil && !errors.Is(err, librbd.ErrNotFound) {
|
||||
log.ErrorLog(ctx, "failed to get metadata: %s. %v", param.rbdQosPerGiBType, err)
|
||||
|
||||
return nil, err
|
||||
}
|
||||
rbdQosParameters[k] = qosSpec{param.rbdQosType, baseLimit, param.rbdQosPerGiBType, perGiBLimit, true}
|
||||
}
|
||||
|
||||
baseVolSize, err := rv.GetMetadata(baseQosVolSize)
|
||||
if err != nil && !errors.Is(err, librbd.ErrNotFound) {
|
||||
log.ErrorLog(ctx, "failed to get metadata: %s. %v", baseQosVolSize, err)
|
||||
|
||||
return nil, err
|
||||
}
|
||||
rv.BaseVolSize = baseVolSize
|
||||
|
||||
return rbdQosParameters, nil
|
||||
}
|
||||
|
||||
func (rv *rbdVolume) AdjustQOS(
|
||||
ctx context.Context,
|
||||
) error {
|
||||
rbdQosParameters, err := rv.getRbdImageQOS(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, param := range rbdQosParameters {
|
||||
err = rv.calcQosBasedOnCapacity(ctx, param)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = rv.ApplyQOS(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
120
internal/rbd/qos_test.go
Normal file
120
internal/rbd/qos_test.go
Normal file
@ -0,0 +1,120 @@
|
||||
/*
|
||||
Copyright 2024 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func checkQOS(
|
||||
t *testing.T,
|
||||
target map[string]string,
|
||||
wants map[string]string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
for k, v := range wants {
|
||||
if r, ok := target[k]; ok {
|
||||
if v != r {
|
||||
t.Errorf("SetQOS: %s: %s, want %s", k, target[k], v)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("SetQOS: missing qos %s", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetQOS(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := context.TODO()
|
||||
|
||||
tests := map[string]string{
|
||||
baseReadIops: "2000",
|
||||
baseWriteIops: "1000",
|
||||
}
|
||||
wants := map[string]string{
|
||||
readIopsLimit: "2000",
|
||||
writeIopsLimit: "1000",
|
||||
}
|
||||
rv := rbdVolume{}
|
||||
rv.RequestedVolSize = int64(oneGB)
|
||||
err := rv.SetQOS(ctx, tests)
|
||||
if err != nil {
|
||||
t.Errorf("SetQOS failed: %v", err)
|
||||
}
|
||||
checkQOS(t, rv.QosParameters, wants)
|
||||
|
||||
tests = map[string]string{
|
||||
baseReadIops: "2000",
|
||||
baseWriteIops: "1000",
|
||||
baseReadBytesPerSecond: "209715200",
|
||||
baseWriteBytesPerSecond: "104857600",
|
||||
}
|
||||
wants = map[string]string{
|
||||
readIopsLimit: "2000",
|
||||
writeIopsLimit: "1000",
|
||||
readBpsLimit: "209715200",
|
||||
writeBpsLimit: "104857600",
|
||||
}
|
||||
rv = rbdVolume{}
|
||||
rv.RequestedVolSize = int64(oneGB)
|
||||
err = rv.SetQOS(ctx, tests)
|
||||
if err != nil {
|
||||
t.Errorf("SetQOS failed: %v", err)
|
||||
}
|
||||
checkQOS(t, rv.QosParameters, wants)
|
||||
|
||||
tests = map[string]string{
|
||||
baseReadIops: "2000",
|
||||
baseWriteIops: "1000",
|
||||
baseReadBytesPerSecond: "209715200",
|
||||
baseWriteBytesPerSecond: "104857600",
|
||||
readIopsPerGiB: "20",
|
||||
writeIopsPerGiB: "10",
|
||||
readBpsPerGiB: "2097152",
|
||||
writeBpsPerGiB: "1048576",
|
||||
baseVolSizeBytes: "21474836480",
|
||||
}
|
||||
wants = map[string]string{
|
||||
readIopsLimit: "2000",
|
||||
writeIopsLimit: "1000",
|
||||
readBpsLimit: "209715200",
|
||||
writeBpsLimit: "104857600",
|
||||
}
|
||||
rv = rbdVolume{}
|
||||
rv.RequestedVolSize = int64(oneGB) * 20
|
||||
err = rv.SetQOS(ctx, tests)
|
||||
if err != nil {
|
||||
t.Errorf("SetQOS failed: %v", err)
|
||||
}
|
||||
checkQOS(t, rv.QosParameters, wants)
|
||||
|
||||
wants = map[string]string{
|
||||
readIopsLimit: "3600",
|
||||
writeIopsLimit: "1800",
|
||||
readBpsLimit: "377487360",
|
||||
writeBpsLimit: "188743680",
|
||||
}
|
||||
rv = rbdVolume{}
|
||||
rv.RequestedVolSize = int64(oneGB) * 100
|
||||
err = rv.SetQOS(ctx, tests)
|
||||
if err != nil {
|
||||
t.Errorf("SetQOS failed: %v", err)
|
||||
}
|
||||
checkQOS(t, rv.QosParameters, wants)
|
||||
}
|
||||
@ -148,6 +148,12 @@ type rbdImage struct {
|
||||
EnableMetadata bool
|
||||
// ParentInTrash indicates the parent image is in trash.
|
||||
ParentInTrash bool
|
||||
|
||||
// RBD QoS configuration
|
||||
QosParameters map[string]string
|
||||
|
||||
// the min size of volume what use to calc qos beased on capacity.
|
||||
BaseVolSize string
|
||||
}
|
||||
|
||||
// check that rbdVolume implements the types.Volume interface.
|
||||
|
||||
Reference in New Issue
Block a user