deploy: support for read affinity options per cluster

Implemented the capability to include read affinity options
for individual clusters within the ceph-csi-config ConfigMap.
This allows users to configure the crush location for each
cluster separately. The read affinity options specified in
the ConfigMap will supersede those provided via command line arguments.

Signed-off-by: Praveen M <m.praveen@ibm.com>

internal/util/crushlocation.go

@@ -23,20 +23,15 @@ import (
 )
 
 // GetCrushLocationMap returns the crush location map, determined from
-// the crush location labels and their values from the CO system.
+// the crush location labels and their values from the node labels passed in arg.
 // Expects crushLocationLabels in arg to be in the format "[prefix/]<name>,[prefix/]<name>,...",.
 // Returns map of crush location types with its array of associated values.
-func GetCrushLocationMap(crushLocationLabels, nodeName string) (map[string]string, error) {
+func GetCrushLocationMap(crushLocationLabels string, nodeLabels map[string]string) map[string]string {
 	if crushLocationLabels == "" {
-		return nil, nil
+		return nil
 	}
 
-	nodeLabels, err := k8sGetNodeLabels(nodeName)
-	if err != nil {
-		return nil, err
-	}
-
-	return getCrushLocationMap(crushLocationLabels, nodeLabels), nil
+	return getCrushLocationMap(crushLocationLabels, nodeLabels)
 }
 
 // getCrushLocationMap returns the crush location map, determined from

internal/util/csiconfig.go

@@ -62,6 +62,11 @@ type ClusterInfo struct {
 		// symlink filepath for the network namespace where we need to execute commands.
 		NetNamespaceFilePath string `json:"netNamespaceFilePath"`
 	} `json:"nfs"`
+	// Read affinity map options
+	ReadAffinity struct {
+		Enabled             bool     `json:"enabled"`
+		CrushLocationLabels []string `json:"crushLocationLabels"`
+	} `json:"readAffinity"`
 }
 
 // Expected JSON structure in the passed in config file is,
@@ -203,3 +208,21 @@ func GetNFSNetNamespaceFilePath(pathToConfig, clusterID string) (string, error)
 
 	return cluster.NFS.NetNamespaceFilePath, nil
 }
+
+// GetCrushLocationLabels returns the `readAffinity.enabled` and `readAffinity.crushLocationLabels`
+// values from the CSI config for the given `clusterID`. If `readAffinity.enabled` is set to true
+// it returns `true` and `crushLocationLabels`, else returns `false` and an empty string.
+func GetCrushLocationLabels(pathToConfig, clusterID string) (bool, string, error) {
+	cluster, err := readClusterInfo(pathToConfig, clusterID)
+	if err != nil {
+		return false, "", err
+	}
+
+	if !cluster.ReadAffinity.Enabled {
+		return false, "", nil
+	}
+
+	crushLocationLabels := strings.Join(cluster.ReadAffinity.CrushLocationLabels, ",")
+
+	return true, crushLocationLabels, nil
+}

internal/util/csiconfig_test.go

@@ -365,3 +365,116 @@ func TestGetNFSNetNamespaceFilePath(t *testing.T) {
 		})
 	}
 }
+
+func TestGetReadAffinityOptions(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name      string
+		clusterID string
+		want      struct {
+			enabled bool
+			labels  string
+		}
+	}{
+		{
+			name:      "ReadAffinity enabled set to true for cluster-1",
+			clusterID: "cluster-1",
+			want: struct {
+				enabled bool
+				labels  string
+			}{true, "topology.kubernetes.io/region,topology.kubernetes.io/zone,topology.io/rack"},
+		},
+		{
+			name:      "ReadAffinity enabled set to true for cluster-2",
+			clusterID: "cluster-2",
+			want: struct {
+				enabled bool
+				labels  string
+			}{true, "topology.kubernetes.io/region"},
+		},
+		{
+			name:      "ReadAffinity enabled set to false for cluster-3",
+			clusterID: "cluster-3",
+			want: struct {
+				enabled bool
+				labels  string
+			}{false, ""},
+		},
+		{
+			name:      "ReadAffinity option not set in cluster-4",
+			clusterID: "cluster-4",
+			want: struct {
+				enabled bool
+				labels  string
+			}{false, ""},
+		},
+	}
+
+	csiConfig := []ClusterInfo{
+		{
+			ClusterID: "cluster-1",
+			ReadAffinity: struct {
+				Enabled             bool     `json:"enabled"`
+				CrushLocationLabels []string `json:"crushLocationLabels"`
+			}{
+				Enabled: true,
+				CrushLocationLabels: []string{
+					"topology.kubernetes.io/region",
+					"topology.kubernetes.io/zone",
+					"topology.io/rack",
+				},
+			},
+		},
+		{
+			ClusterID: "cluster-2",
+			ReadAffinity: struct {
+				Enabled             bool     `json:"enabled"`
+				CrushLocationLabels []string `json:"crushLocationLabels"`
+			}{
+				Enabled: true,
+				CrushLocationLabels: []string{
+					"topology.kubernetes.io/region",
+				},
+			},
+		},
+		{
+			ClusterID: "cluster-3",
+			ReadAffinity: struct {
+				Enabled             bool     `json:"enabled"`
+				CrushLocationLabels []string `json:"crushLocationLabels"`
+			}{
+				Enabled: false,
+				CrushLocationLabels: []string{
+					"topology.io/rack",
+				},
+			},
+		},
+		{
+			ClusterID: "cluster-4",
+		},
+	}
+	csiConfigFileContent, err := json.Marshal(csiConfig)
+	if err != nil {
+		t.Errorf("failed to marshal csi config info %v", err)
+	}
+	tmpConfPath := t.TempDir() + "/ceph-csi.json"
+	err = os.WriteFile(tmpConfPath, csiConfigFileContent, 0o600)
+	if err != nil {
+		t.Errorf("failed to write %s file content: %v", CsiConfigFile, err)
+	}
+	for _, tt := range tests {
+		tc := tt
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			enabled, labels, err := GetCrushLocationLabels(tmpConfPath, tc.clusterID)
+			if err != nil {
+				t.Errorf("GetCrushLocationLabels() error = %v", err)
+
+				return
+			}
+			if enabled != tc.want.enabled || labels != tc.want.labels {
+				t.Errorf("GetCrushLocationLabels() = {%v %v} want %v", enabled, labels, tc.want)
+			}
+		})
+	}
+}