rbd: add capability to automatically enable read affinity

This commit makes use of crush location labels from node
labels to supply `crush_location` and `read_from_replica=localize`
options during rbd map cmd. Using these options, ceph
will be able to redirect reads to the closest OSD,
improving performance.

Signed-off-by: Rakshith R <rar@redhat.com>

internal/util/crushlocation.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2023 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"strings"
+
+	"github.com/ceph/ceph-csi/internal/util/log"
+)
+
+// GetCrushLocationMap returns the crush location map, determined from
+// the crush location labels and their values from the CO system.
+// Expects crushLocationLabels in arg to be in the format "[prefix/]<name>,[prefix/]<name>,...",.
+// Returns map of crush location types with its array of associated values.
+func GetCrushLocationMap(crushLocationLabels, nodeName string) (map[string]string, error) {
+	if crushLocationLabels == "" {
+		return nil, nil
+	}
+
+	nodeLabels, err := k8sGetNodeLabels(nodeName)
+	if err != nil {
+		return nil, err
+	}
+
+	return getCrushLocationMap(crushLocationLabels, nodeLabels), nil
+}
+
+// getCrushLocationMap returns the crush location map, determined from
+// the crush location labels and node labels.
+func getCrushLocationMap(crushLocationLabels string, nodeLabels map[string]string) map[string]string {
+	labelsToRead := strings.Split(crushLocationLabels, labelSeparator)
+	log.DefaultLog("CRUSH location labels passed for processing: %+v", labelsToRead)
+
+	labelsIn := make(map[string]bool, len(labelsToRead))
+	for _, label := range labelsToRead {
+		labelsIn[label] = true
+	}
+
+	// Determine values for requested labels from node labels
+	crushLocationMap := make(map[string]string, len(labelsIn))
+	for key, value := range nodeLabels {
+		if _, ok := labelsIn[key]; !ok {
+			continue
+		}
+		// label found split name component and store value
+		nameIdx := strings.IndexRune(key, keySeparator)
+		crushLocationType := strings.TrimSpace(key[nameIdx+1:])
+		if crushLocationType == "hostname" {
+			// ceph defaults to "host" while Kubernetes uses "hostname" as key.
+			crushLocationType = "host"
+		}
+		// replace "." with "-" to satisfy ceph crush map.
+		value = strings.Replace(strings.TrimSpace(value), ".", "-", -1)
+		crushLocationMap[crushLocationType] = value
+	}
+
+	if len(crushLocationMap) == 0 {
+		return nil
+	}
+	log.DefaultLog("list of CRUSH location processed: %+v", crushLocationMap)
+
+	return crushLocationMap
+}

internal/util/crushlocation_test.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2023 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_getCrushLocationMap(t *testing.T) {
+	t.Parallel()
+	type input struct {
+		crushLocationLabels string
+		nodeLabels          map[string]string
+	}
+	tests := []struct {
+		name string
+		args input
+		want map[string]string
+	}{
+		{
+			name: "empty crushLocationLabels",
+			args: input{
+				crushLocationLabels: "",
+				nodeLabels:          map[string]string{},
+			},
+			want: nil,
+		},
+		{
+			name: "empty nodeLabels",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/rack",
+				nodeLabels:          map[string]string{},
+			},
+			want: nil,
+		},
+		{
+			name: "matching crushlocation and node labels",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/rack",
+				nodeLabels: map[string]string{
+					"topology.io/zone": "zone1",
+				},
+			},
+			want: map[string]string{"zone": "zone1"},
+		},
+		{
+			name: "multuple matching crushlocation and node labels",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/rack",
+				nodeLabels: map[string]string{
+					"topology.io/zone": "zone1",
+					"topology.io/rack": "rack1",
+				},
+			},
+			want: map[string]string{"zone": "zone1", "rack": "rack1"},
+		},
+		{
+			name: "no match between crushlocation and node labels",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/rack",
+				nodeLabels: map[string]string{
+					"topology.io/region": "region1",
+				},
+			},
+			want: nil,
+		},
+		{
+			name: "check crushlocation value replacement to satisfy ceph requirement",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/rack",
+				nodeLabels: map[string]string{
+					"topology.io/zone": "south.east.1",
+				},
+			},
+			want: map[string]string{"zone": "south-east-1"},
+		},
+		{
+			name: "hostname key should be replaced with host",
+			args: input{
+				crushLocationLabels: "topology.io/zone,topology.io/hostname",
+				nodeLabels: map[string]string{
+					"topology.io/hostname": "worker-1",
+				},
+			},
+			want: map[string]string{"host": "worker-1"},
+		},
+	}
+	for _, tt := range tests {
+		currentTT := tt
+		t.Run(currentTT.name, func(t *testing.T) {
+			t.Parallel()
+			assert.Equal(t,
+				currentTT.want,
+				getCrushLocationMap(currentTT.args.crushLocationLabels, currentTT.args.nodeLabels))
+		})
+	}
+}

internal/util/util.go

@@ -147,6 +147,10 @@ type Config struct {
 
 	// Cluster name
 	ClusterName string
+
+	// Read affinity related options
+	EnableReadAffinity  bool   // enable OSD read affinity.
+	CrushLocationLabels string // list of CRUSH location labels to read from the node.
 }
 
 // ValidateDriverName validates the driver name.