From c83a2818578bbfaa50f5ea148e94b075533ca908 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 18 Apr 2022 12:02:31 +0530 Subject: [PATCH] cephfs: add netNamespaceFilePath for CephFS as same host directory is not shared between the cephfs and the rbd plugin pod. we need to keep the netNamespaceFilePath separately for both cephfs and rbd. CephFS plugin will use this path to execute mount -t commands. Signed-off-by: Madhu Rajanna (cherry picked from commit d2bc9743f751d60c24db88a87f288ca48e945308) --- charts/ceph-csi-cephfs/values.yaml | 2 +- examples/csi-config-map-sample.yaml | 5 ++ internal/cephfs/nodeserver.go | 6 +++ internal/util/csiconfig.go | 12 +++++ internal/util/csiconfig_test.go | 76 +++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 1 deletion(-) diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 07f54ad15..9822480b4 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -27,7 +27,7 @@ serviceAccounts: # - "" # cephFS: # subvolumeGroup: "csi" -# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" +# netNamespaceFilePath: "{{ .kubeletDir }}/plugins/{{ .driverName }}/net" csiConfig: [] # Set logging level for csi containers. diff --git a/examples/csi-config-map-sample.yaml b/examples/csi-config-map-sample.yaml index ee8c6a63d..27909d743 100644 --- a/examples/csi-config-map-sample.yaml +++ b/examples/csi-config-map-sample.yaml @@ -20,6 +20,10 @@ kind: ConfigMap # NOTE: Make sure you don't add radosNamespace option to a currently in use # configuration as it will cause issues. # The field "cephFS.subvolumeGroup" is optional and defaults to "csi". +# The "cephFS.netNamespaceFilePath" fields are the various network namespace +# path for the Ceph cluster identified by the , This will be used +# by the CephFS CSI plugin to execute the mount -t in the +# network namespace specified by the "cephFS.netNamespaceFilePath". # The "rbd.netNamespaceFilePath" fields are the various network namespace # path for the Ceph cluster identified by the , This will be used # by the RBD CSI plugin to execute the rbd map/unmap in the @@ -54,6 +58,7 @@ data: ], "cephFS": { "subvolumeGroup": "" + "netNamespaceFilePath": "/plugins/cephfs.csi.ceph.com/net", } } ] diff --git a/internal/cephfs/nodeserver.go b/internal/cephfs/nodeserver.go index 21457ff38..fdadb6497 100644 --- a/internal/cephfs/nodeserver.go +++ b/internal/cephfs/nodeserver.go @@ -126,6 +126,12 @@ func (ns *NodeServer) NodeStageVolume( } defer volOptions.Destroy() + volOptions.NetNamespaceFilePath, err = util.GetCephFSNetNamespaceFilePath( + util.CsiConfigFile, + volOptions.ClusterID) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } mnt, err := mounter.New(volOptions) if err != nil { log.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err) diff --git a/internal/util/csiconfig.go b/internal/util/csiconfig.go index 1b45acf10..bdfe6d267 100644 --- a/internal/util/csiconfig.go +++ b/internal/util/csiconfig.go @@ -46,6 +46,8 @@ type ClusterInfo struct { Monitors []string `json:"monitors"` // CephFS contains CephFS specific options CephFS struct { + // symlink filepath for the network namespace where we need to execute commands. + NetNamespaceFilePath string `json:"netNamespaceFilePath"` // SubvolumeGroup contains the name of the SubvolumeGroup for CSI volumes SubvolumeGroup string `json:"subvolumeGroup"` } `json:"cephFS"` @@ -182,3 +184,13 @@ func GetRBDNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) return cluster.RBD.NetNamespaceFilePath, nil } + +// GetCephFSNetNamespaceFilePath returns the netNamespaceFilePath for CephFS volumes. +func GetCephFSNetNamespaceFilePath(pathToConfig, clusterID string) (string, error) { + cluster, err := readClusterInfo(pathToConfig, clusterID) + if err != nil { + return "", err + } + + return cluster.CephFS.NetNamespaceFilePath, nil +} diff --git a/internal/util/csiconfig_test.go b/internal/util/csiconfig_test.go index 519b25011..4a37e588b 100644 --- a/internal/util/csiconfig_test.go +++ b/internal/util/csiconfig_test.go @@ -215,3 +215,79 @@ func TestGetRBDNetNamespaceFilePath(t *testing.T) { }) } } + +func TestGetCephFSNetNamespaceFilePath(t *testing.T) { + t.Parallel() + tests := []struct { + name string + clusterID string + want string + }{ + { + name: "get cephFS specific NetNamespaceFilePath for cluster-1", + clusterID: "cluster-1", + want: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster1-net", + }, + { + name: "get cephFS specific NetNamespaceFilePath for cluster-2", + clusterID: "cluster-2", + want: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster2-net", + }, + { + name: "when cephFS specific NetNamespaceFilePath is empty", + clusterID: "cluster-3", + want: "", + }, + } + + csiConfig := []ClusterInfo{ + { + ClusterID: "cluster-1", + Monitors: []string{"ip-1", "ip-2"}, + CephFS: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + SubvolumeGroup string `json:"subvolumeGroup"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster1-net", + }, + }, + { + ClusterID: "cluster-2", + Monitors: []string{"ip-3", "ip-4"}, + CephFS: struct { + NetNamespaceFilePath string `json:"netNamespaceFilePath"` + SubvolumeGroup string `json:"subvolumeGroup"` + }{ + NetNamespaceFilePath: "/var/lib/kubelet/plugins/cephfs.ceph.csi.com/cluster2-net", + }, + }, + { + ClusterID: "cluster-3", + Monitors: []string{"ip-5", "ip-6"}, + }, + } + csiConfigFileContent, err := json.Marshal(csiConfig) + if err != nil { + t.Errorf("failed to marshal csi config info %v", err) + } + tmpConfPath := t.TempDir() + "/ceph-csi.json" + err = os.WriteFile(tmpConfPath, csiConfigFileContent, 0o600) + if err != nil { + t.Errorf("failed to write %s file content: %v", CsiConfigFile, err) + } + for _, tt := range tests { + ts := tt + t.Run(ts.name, func(t *testing.T) { + t.Parallel() + got, err := GetCephFSNetNamespaceFilePath(tmpConfPath, ts.clusterID) + if err != nil { + t.Errorf("GetCephFSNetNamespaceFilePath() error = %v", err) + + return + } + if got != ts.want { + t.Errorf("GetCephFSNetNamespaceFilePath() = %v, want %v", got, ts.want) + } + }) + } +}