mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2025-06-13 10:33:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

rbd: support QoS based on capacity for rbd volume

Browse Source 
1. QoS provides settings for rbd volume read/write iops
   and read/write bandwidth.
2. All QoS parameters are placed in the SC,
   send QoS parameters from SC to Cephcsi through PVC create request.
3. We need provide QoS parameters in the SC as below:
   - BaseReadIops
   - BaseWriteIops
   - BaseReadBytesPerSecond
   - BaseWriteBytesPerSecond
   - ReadIopsPerGB
   - WriteIopsPerGB
   - ReadBpsPerGB
   - WriteBpsPerGB
   - BaseVolSizeBytes
   There are 4 base qos parameters among them, when users apply for
   a volume capacity equal to or less than BaseVolSizebytes, use base
   qos limit. For the portion of capacity exceeding BaseVolSizebytes,
   QoS will be increased in steps set per GB. If the step size parameter
   per GB is not provided, only base QoS limit will be used and not associated
   with capacity size.
4. If PVC has resize request, adjust the QoS limit
   according to the QoS parameters after resizing.

Signed-off-by: Yite Gu <guyite@bytedance.com>
This commit is contained in:
Yite Gu
2024-12-12 17:26:25 +08:00
committed by mergify[bot]
parent e4d41c42d6
commit 7595e20969
9 changed files with 778 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
e2e/rbd.go
View File

@ -27,6 +27,7 @@ import (
. "github.com/onsi/ginkgo/v2"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
@ -4646,6 +4647,284 @@ var _ = Describe("RBD", func() {
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})
By("validate rbd image qos", func() {
var (
baseReadIops = "2000"
baseWriteIops = "1000"
baseReadBytesPerSecond = "209715200"
baseWriteBytesPerSecond = "104857600"
readIopsPerGiB = "20"
writeIopsPerGiB = "10"
readBpsPerGiB = "2097152"
writeBpsPerGiB = "1048576"
baseVolSizeBytes = "21474836480"
)
qosParameters := map[string]string{
"BaseReadIops": baseReadIops,
"BaseWriteIops": baseWriteIops,
"BaseReadBytesPerSecond": baseReadBytesPerSecond,
"BaseWriteBytesPerSecond": baseWriteBytesPerSecond,
}
err := deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
}()
// 1.1 create PVC
pvc, err := loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
// 1.2 validate rbd image qos
wants := map[string]string{
"rbd_qos_read_iops_limit": baseReadIops,
"rbd_qos_write_iops_limit": baseWriteIops,
"rbd_qos_read_bps_limit": baseReadBytesPerSecond,
"rbd_qos_write_bps_limit": baseWriteBytesPerSecond,
}
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 1.3 delete pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
qosParameters = map[string]string{
"BaseReadIops": baseReadIops,
"BaseWriteIops": baseWriteIops,
"BaseReadBytesPerSecond": baseReadBytesPerSecond,
"BaseWriteBytesPerSecond": baseWriteBytesPerSecond,
"ReadIopsPerGiB": readIopsPerGiB,
"WriteIopsPerGiB": writeIopsPerGiB,
"ReadBpsPerGiB": readBpsPerGiB,
"WriteBpsPerGiB": writeBpsPerGiB,
"BaseVolSizeBytes": baseVolSizeBytes,
}
err = deleteResource(rbdExamplePath + "storageclass.yaml")
if err != nil {
framework.Failf("failed to delete storageclass: %v", err)
}
err = createRBDStorageClass(
f.ClientSet,
f,
defaultSCName,
nil,
qosParameters,
deletePolicy)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
// 2.1 create PVC
pvc, err = loadPVC(pvcPath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvc.Namespace = f.UniqueName
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC and application: %v", err)
}
// validate created backend rbd images
validateRBDImageCount(f, 1, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
// 2.2 validate rbd image qos
wants = map[string]string{
"rbd_qos_read_iops_limit": "3600",
"rbd_qos_write_iops_limit": "1800",
"rbd_qos_read_bps_limit": "377487360",
"rbd_qos_write_bps_limit": "188743680",
}
err = validateQOS(f, pvc, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 3.1 create snapshot
err = createRBDSnapshotClass(f)
if err != nil {
framework.Failf("failed to create storageclass: %v", err)
}
defer func() {
err = deleteRBDSnapshotClass()
if err != nil {
framework.Failf("failed to delete VolumeSnapshotClass: %v", err)
}
}()
snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
err = createSnapshot(&snap, deployTimeout)
if err != nil {
framework.Failf("failed to create snapshot: %v", err)
}
// validate created backend rbd images
// parent PVC + snapshot
totalImages := 2
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 1, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
// 3.2 create pvc from snapshot
pvcClone, err := loadPVC(pvcClonePath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC: %v", err)
}
// validate created backend rbd images
// parent pvc + snapshot + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
validateOmapCount(f, 1, rbdType, defaultRBDPool, snapsType)
// 3.3 validate rbd image qos
err = validateQOS(f, pvcClone, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 3.4 delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
// 3.5 validate create pvc from snapshot, but pvc size greater than parent
pvcClone, err = loadPVC(pvcClonePath)
if err != nil {
framework.Failf("failed to load PVC: %v", err)
}
pvcClone.Namespace = f.UniqueName
pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("200Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to create PVC: %v", err)
}
wants2 := map[string]string{
"rbd_qos_read_iops_limit": "5600",
"rbd_qos_write_iops_limit": "2800",
"rbd_qos_read_bps_limit": "587202560",
"rbd_qos_write_bps_limit": "293601280",
}
err = validateQOS(f, pvcClone, wants2)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 3.6 delete snapshot and clone pvc
err = deleteSnapshot(&snap, deployTimeout)
if err != nil {
framework.Failf("failed to delete snapshot: %v", err)
}
err = deletePVCAndValidatePV(f.ClientSet, pvcClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
// 4.1 create pvc from pvc
pvcSmartClone, err := loadPVC(pvcSmartClonePath)
if err != nil {
framework.Failf("failed to load pvcSmartClone: %v", err)
}
pvcSmartClone.Namespace = f.UniqueName
pvcSmartClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("100Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to create pvc: %v", err)
}
// validate created backend rbd images
// parent pvc + temp clone + clone
totalImages = 3
validateRBDImageCount(f, totalImages, defaultRBDPool)
validateOmapCount(f, 2, rbdType, defaultRBDPool, volumesType)
// 4.2 validate rbd image qos
err = validateQOS(f, pvcSmartClone, wants)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 4.3 delete clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
// 4.4 create pvc from pvc, but pvc size greater than parent
pvcSmartClone, err = loadPVC(pvcSmartClonePath)
if err != nil {
framework.Failf("failed to load pvcSmartClone: %v", err)
}
pvcSmartClone.Namespace = f.UniqueName
pvcSmartClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse("200Gi")
err = createPVCAndvalidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to create pvc: %v", err)
}
err = validateQOS(f, pvcSmartClone, wants2)
if err != nil {
framework.Failf("failed to validate qos: %v", err)
}
// 4.5 delete parent pvc and clone pvc
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
err = deletePVCAndValidatePV(f.ClientSet, pvcSmartClone, deployTimeout)
if err != nil {
framework.Failf("failed to delete PVC: %v", err)
}
// END: validate created backend rbd images
validateRBDImageCount(f, 0, defaultRBDPool)
validateOmapCount(f, 0, rbdType, defaultRBDPool, volumesType)
})
By("create a PVC and check PVC/PV metadata on RBD image after setmetadata is set to false", func() {
err := createRBDSnapshotClass(f)
if err != nil {

25
e2e/rbd_helper.go
View File

@ -1166,3 +1166,28 @@ func validateStripe(f *framework.Framework,
return nil
}
func validateQOS(f *framework.Framework,
pvc *v1.PersistentVolumeClaim,
wants map[string]string,
) error {
metadataConfPrefix := "conf_"
imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f)
if err != nil {
return err
}
rbdImageSpec := imageSpec(defaultRBDPool, imageData.imageName)
for k, v := range wants {
qosVal, err := getImageMeta(rbdImageSpec, metadataConfPrefix+k, f)
if err != nil {
return err
}
if qosVal != v {
return fmt.Errorf("%s: %s does not match expected %s", k, qosVal, v)
}
}
return nil
}