From 9d5ee285a24ccbc33f384085dd4b4a1a36ae8caf Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Mon, 11 Sep 2023 11:50:54 +0530 Subject: [PATCH 1/8] cephfs: adds the implementation of client eviction this commit adds client eviction to cephfs, based on the IPs in cidr block, it evicts those IPs from the network. Signed-off-by: Riya Singhal --- internal/csi-addons/cephfs/network_fence.go | 91 ++++++++++ internal/csi-addons/networkfence/fencing.go | 157 +++++++++++++++++- .../csi-addons/networkfence/fencing_test.go | 5 +- 3 files changed, 251 insertions(+), 2 deletions(-) create mode 100644 internal/csi-addons/cephfs/network_fence.go diff --git a/internal/csi-addons/cephfs/network_fence.go b/internal/csi-addons/cephfs/network_fence.go new file mode 100644 index 000000000..fe4f93af5 --- /dev/null +++ b/internal/csi-addons/cephfs/network_fence.go @@ -0,0 +1,91 @@ +/* +Copyright 2023 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cephfs + +import ( + "context" + "errors" + + nf "github.com/ceph/ceph-csi/internal/csi-addons/networkfence" + "github.com/ceph/ceph-csi/internal/util" + + "github.com/csi-addons/spec/lib/go/fence" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// FenceControllerServer struct of cephFS CSI driver with supported methods +// of CSI-Addons networkfence controller service spec. +type FenceControllerServer struct { + *fence.UnimplementedFenceControllerServer +} + +// NewFenceControllerServer creates a new FenceControllerServer which handles +// the FenceController Service requests from the CSI-Addons specification. +func NewFenceControllerServer() *FenceControllerServer { + return &FenceControllerServer{} +} + +// RegisterService registers the FenceControllerServer's service +// with the gRPC server. +func (fcs *FenceControllerServer) RegisterService(server grpc.ServiceRegistrar) { + fence.RegisterFenceControllerServer(server, fcs) +} + +// validateFenceClusterNetworkReq checks the sanity of FenceClusterNetworkRequest. +func validateNetworkFenceReq(fenceClients []*fence.CIDR, options map[string]string) error { + if len(fenceClients) == 0 { + return errors.New("CIDR block cannot be empty") + } + + if value, ok := options["clusterID"]; !ok || value == "" { + return errors.New("missing or empty clusterID") + } + + return nil +} + +// FenceClusterNetwork blocks access to a CIDR block by creating a network fence. +// It evicts the IP addresses of clients, which are in CIDR block. +func (fcs *FenceControllerServer) FenceClusterNetwork( + ctx context.Context, + req *fence.FenceClusterNetworkRequest, +) (*fence.FenceClusterNetworkResponse, error) { + err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + cr, err := util.NewUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + defer cr.DeleteCredentials() + + nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + err = nwFence.AddClientEviction(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to fence CIDR block %q: %s", nwFence.Cidr, err.Error()) + } + + return &fence.FenceClusterNetworkResponse{}, nil +} diff --git a/internal/csi-addons/networkfence/fencing.go b/internal/csi-addons/networkfence/fencing.go index 60c3e61fa..205f85ffc 100644 --- a/internal/csi-addons/networkfence/fencing.go +++ b/internal/csi-addons/networkfence/fencing.go @@ -1,9 +1,12 @@ /* -Copyright 2022 The Ceph-CSI Authors. +Copyright 2023 The Ceph-CSI Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,10 +18,13 @@ package networkfence import ( "context" + "encoding/json" "errors" "fmt" "net" + "strconv" "strings" + "time" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -29,6 +35,8 @@ import ( const ( blocklistTime = "157784760" invalidCommandStr = "invalid command" + // we can always use mds rank 0, since all the clients have a session with rank-0. + mdsRank = "0" ) // NetworkFence contains the CIDR blocks to be blocked. @@ -38,6 +46,11 @@ type NetworkFence struct { cr *util.Credentials } +// activeClient represents the structure of an active client. +type activeClient struct { + Inst string `json:"inst"` +} + // NewNetworkFence returns a networkFence struct object from the Network fence/unfence request. func NewNetworkFence( ctx context.Context, @@ -132,6 +145,148 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { return nil } +func listActiveClients(ctx context.Context) ([]activeClient, error) { + // FIXME: replace the ceph command with go-ceph API in future + cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "ls"} + stdout, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...) + if err != nil { + return nil, fmt.Errorf("failed to list active clients: %w, stderr: %q", err, stdErr) + } + + var activeClients []activeClient + if err := json.Unmarshal([]byte(stdout), &activeClients); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON: %w", err) + } + + return activeClients, nil +} + +func evictCephFSClient(ctx context.Context, clientID int) error { + // FIXME: replace the ceph command with go-ceph API in future + cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)} + _, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...) + if err != nil { + return fmt.Errorf("failed to evict client %d: %w, stderr: %q", clientID, err, stdErr) + } + log.DebugLog(ctx, "client %s has been evicted from CephFS\n", clientID) + + return nil +} + +func isIPInCIDR(ctx context.Context, ip, cidr string) bool { + // Parse the CIDR block + _, ipCidr, err := net.ParseCIDR(cidr) + if err != nil { + log.ErrorLog(ctx, "error parsing CIDR block %s: %w\n", cidr, err) + + return false + } + + // Parse the IP address + ipAddress := net.ParseIP(ip) + if ipAddress == nil { + log.ErrorLog(ctx, "error parsing IP address %s\n", ip) + + return false + } + + // Check if the IP address is within the CIDR block + return ipCidr.Contains(ipAddress) +} + +func (ac *activeClient) fetchIP() (string, error) { + // example: "inst": "client.4305 172.21.9.34:0/422650892", + // then returning value will be 172.21.9.34 + clientInfo := ac.Inst + parts := strings.Fields(clientInfo) + if len(parts) >= 2 { + ip := strings.Split(parts[1], ":")[0] + + return ip, nil + } + + return "", fmt.Errorf("failed to extract IP address, incorrect format: %s", clientInfo) +} + +func (ac *activeClient) fetchID() (int, error) { + // example: "inst": "client.4305 172.21.9.34:0/422650892", + // then returning value will be 4305 + clientInfo := ac.Inst + parts := strings.Fields(clientInfo) + if len(parts) >= 1 { + clientIDStr := strings.TrimPrefix(parts[0], "client.") + clientID, err := strconv.Atoi(clientIDStr) + if err != nil { + return 0, fmt.Errorf("failed to convert client ID to int: %w", err) + } + + return clientID, nil + } + + return 0, fmt.Errorf("failed to extract client ID, incorrect format: %s", clientInfo) + +} + +// AddClientEviction blocks access for all the IPs in the CIDR block +// using client eviction. +// blocks the active clients listed in cidr, and the IPs +// for whom there is no active client present too. +func (nf *NetworkFence) AddClientEviction(ctx context.Context) error { + evictedIPs := make(map[string]bool) + // fetch active clients + activeClients, err := listActiveClients(ctx) + if err != nil { + return err + } + // iterate through CIDR blocks and check if any active client matches + for _, cidr := range nf.Cidr { + for _, client := range activeClients { + clientIP, err := client.fetchIP() + if err != nil { + return fmt.Errorf("error fetching client IP: %w", err) + } + // check if the clientIP is in the CIDR block + if isIPInCIDR(ctx, clientIP, cidr) { + clientID, err := client.fetchID() + if err != nil { + return fmt.Errorf("error fetching client ID: %w", err) + } + // evict the client + err = evictCephFSClient(ctx, clientID) + if err != nil { + return fmt.Errorf("error evicting client %d: %w", clientID, err) + } + log.DebugLog(ctx, "client %d has been evicted\n", clientID) + // add the CIDR to the list of blocklisted IPs + evictedIPs[clientIP] = true + } + } + } + + // blocklist the IPs in CIDR without any active clients + for _, cidr := range nf.Cidr { + // check if the CIDR is evicted + // fetch the list of IPs from a CIDR block + hosts, err := getIPRange(cidr) + if err != nil { + return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range: %w", cidr, err) + } + + // add ceph blocklist for each IP in the range mentioned by the CIDR + for _, host := range hosts { + if evictedIPs[host] { + continue + } + err = nf.addCephBlocklist(ctx, host, false) + if err != nil { + return err + } + } + } + + return nil +} + // getIPRange returns a list of IPs from the IP range // corresponding to a CIDR block. func getIPRange(cidr string) ([]string, error) { diff --git a/internal/csi-addons/networkfence/fencing_test.go b/internal/csi-addons/networkfence/fencing_test.go index 6803c994c..86bd34984 100644 --- a/internal/csi-addons/networkfence/fencing_test.go +++ b/internal/csi-addons/networkfence/fencing_test.go @@ -1,9 +1,12 @@ /* -Copyright 2022 The Ceph-CSI Authors. +Copyright 2023 The Ceph-CSI Authors. + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 6b0c4129964424ce0af49b794411045d3284c663 Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Mon, 11 Sep 2023 14:57:16 +0530 Subject: [PATCH 2/8] cephfs: add network fence service to identity.go Signed-off-by: Riya Singhal --- internal/csi-addons/cephfs/identity.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/csi-addons/cephfs/identity.go b/internal/csi-addons/cephfs/identity.go index c869a94f8..fb02f1f37 100644 --- a/internal/csi-addons/cephfs/identity.go +++ b/internal/csi-addons/cephfs/identity.go @@ -77,6 +77,12 @@ func (is *IdentityServer) GetCapabilities( Type: identity.Capability_Service_CONTROLLER_SERVICE, }, }, + }, &identity.Capability{ + Type: &identity.Capability_NetworkFence_{ + NetworkFence: &identity.Capability_NetworkFence{ + Type: identity.Capability_NetworkFence_NETWORK_FENCE, + }, + }, }) } From b8e74e62c0387c3d2bc99ac3828ee78b0ae83141 Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Tue, 26 Sep 2023 01:11:04 +0530 Subject: [PATCH 3/8] cephfs: adding unit test for fetchIP for client eviction Signed-off-by: Riya Singhal --- .../csi-addons/networkfence/fencing_test.go | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/internal/csi-addons/networkfence/fencing_test.go b/internal/csi-addons/networkfence/fencing_test.go index 86bd34984..31623f083 100644 --- a/internal/csi-addons/networkfence/fencing_test.go +++ b/internal/csi-addons/networkfence/fencing_test.go @@ -54,3 +54,43 @@ func TestGetIPRange(t *testing.T) { }) } } + +func TestFetchIP(t *testing.T) { + t.Parallel() + + tests := []struct { + clientInfo string + expectedIP string + expectedErr bool + }{ + { + clientInfo: "client.4305 172.21.9.34:0/422650892", + expectedIP: "172.21.9.34", + expectedErr: false, + }, + { + clientInfo: "", + expectedIP: "", + expectedErr: true, + }, + } + + for _, tt := range tests { + ts := tt + + t.Run(ts.clientInfo, func(t *testing.T) { + t.Parallel() + + client := activeClient{Inst: ts.clientInfo} + ip, actualErr := client.fetchIP() + + if (actualErr != nil) != ts.expectedErr { + t.Errorf("expected error %v but got %v", ts.expectedErr, actualErr) + } + + if ip != ts.expectedIP { + t.Errorf("expected IP %s but got %s", ts.expectedIP, ip) + } + }) + } +} From d925937d5308a7f10b66e378c51b3f80284fd17d Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Tue, 26 Sep 2023 02:18:21 +0530 Subject: [PATCH 4/8] cephfs: adding unit test for fetchID Signed-off-by: Riya Singhal --- internal/csi-addons/networkfence/fencing.go | 1 - .../csi-addons/networkfence/fencing_test.go | 38 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/internal/csi-addons/networkfence/fencing.go b/internal/csi-addons/networkfence/fencing.go index 205f85ffc..eeda40683 100644 --- a/internal/csi-addons/networkfence/fencing.go +++ b/internal/csi-addons/networkfence/fencing.go @@ -224,7 +224,6 @@ func (ac *activeClient) fetchID() (int, error) { } return 0, fmt.Errorf("failed to extract client ID, incorrect format: %s", clientInfo) - } // AddClientEviction blocks access for all the IPs in the CIDR block diff --git a/internal/csi-addons/networkfence/fencing_test.go b/internal/csi-addons/networkfence/fencing_test.go index 31623f083..bbe82120d 100644 --- a/internal/csi-addons/networkfence/fencing_test.go +++ b/internal/csi-addons/networkfence/fencing_test.go @@ -94,3 +94,41 @@ func TestFetchIP(t *testing.T) { }) } } + +func TestFetchID(t *testing.T) { + t.Parallel() + + tests := []struct { + clientInfo string + expectedID int + expectedErr bool + }{ + { + clientInfo: "client.4305 172.21.9.34:0/422650892", + expectedID: 4305, + expectedErr: false, + }, + { + clientInfo: "", + expectedID: 0, + expectedErr: true, + }, + } + + for _, tt := range tests { + ts := tt + t.Run(ts.clientInfo, func(t *testing.T) { + t.Parallel() + ac := &activeClient{Inst: ts.clientInfo} + actualID, actualErr := ac.fetchID() + + if (actualErr != nil) != ts.expectedErr { + t.Errorf("expected error %v but got %v", ts.expectedErr, actualErr) + } + + if actualID != ts.expectedID { + t.Errorf("expected ID %d but got %d", ts.expectedID, actualID) + } + }) + } +} From 1420ad193ab3b3021b885fcd57a697feac67a758 Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Tue, 26 Sep 2023 12:31:51 +0530 Subject: [PATCH 5/8] cephfs: adds unit testing for fencing procedure Signed-off-by: Riya Singhal --- .../csi-addons/cephfs/network_fence_test.go | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 internal/csi-addons/cephfs/network_fence_test.go diff --git a/internal/csi-addons/cephfs/network_fence_test.go b/internal/csi-addons/cephfs/network_fence_test.go new file mode 100644 index 000000000..ce463f134 --- /dev/null +++ b/internal/csi-addons/cephfs/network_fence_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2023 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cephfs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/csi-addons/spec/lib/go/fence" +) + +// TestFenceClusterNetwork is a minimal test for the FenceClusterNetwork() +// procedure. During unit-testing, there is no Ceph cluster available, so +// actual operations can not be performed. +func TestFenceClusterNetwork(t *testing.T) { + t.Parallel() + + controller := NewFenceControllerServer() + + req := &fence.FenceClusterNetworkRequest{ + Parameters: map[string]string{}, + Secrets: nil, + Cidrs: nil, + } + + _, err := controller.FenceClusterNetwork(context.TODO(), req) + assert.Error(t, err) +} From 14b06837d0ff1e537a3f045250fc7208df7f469c Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Fri, 29 Sep 2023 01:26:05 +0530 Subject: [PATCH 6/8] cephfs: implement the logic for unfencing procedure this commit un-blocklists the clients provided in cidr for unfencing operation. Signed-off-by: Riya Singhal --- internal/csi-addons/cephfs/network_fence.go | 29 ++++++++++++++++++ internal/csi-addons/networkfence/fencing.go | 33 ++++++++++++++++----- 2 files changed, 55 insertions(+), 7 deletions(-) diff --git a/internal/csi-addons/cephfs/network_fence.go b/internal/csi-addons/cephfs/network_fence.go index fe4f93af5..c40083392 100644 --- a/internal/csi-addons/cephfs/network_fence.go +++ b/internal/csi-addons/cephfs/network_fence.go @@ -89,3 +89,32 @@ func (fcs *FenceControllerServer) FenceClusterNetwork( return &fence.FenceClusterNetworkResponse{}, nil } + +// UnfenceClusterNetwork unblocks the access to a CIDR block by removing the network fence. +func (fcs *FenceControllerServer) UnfenceClusterNetwork( + ctx context.Context, + req *fence.UnfenceClusterNetworkRequest, +) (*fence.UnfenceClusterNetworkResponse, error) { + err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + cr, err := util.NewUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + defer cr.DeleteCredentials() + + nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + err = nwFence.RemoveNetworkFence(ctx) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to unfence CIDR block %q: %s", nwFence.Cidr, err.Error()) + } + + return &fence.UnfenceClusterNetworkResponse{}, nil +} diff --git a/internal/csi-addons/networkfence/fencing.go b/internal/csi-addons/networkfence/fencing.go index eeda40683..3f7725cab 100644 --- a/internal/csi-addons/networkfence/fencing.go +++ b/internal/csi-addons/networkfence/fencing.go @@ -36,7 +36,7 @@ const ( blocklistTime = "157784760" invalidCommandStr = "invalid command" // we can always use mds rank 0, since all the clients have a session with rank-0. - mdsRank = "0" + mdsRank = 0 ) // NetworkFence contains the CIDR blocks to be blocked. @@ -145,9 +145,15 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { return nil } -func listActiveClients(ctx context.Context) ([]activeClient, error) { +func (nf *NetworkFence) listActiveClients(ctx context.Context) ([]activeClient, error) { + arg := []string{ + "--id", nf.cr.ID, + "--keyfile=" + nf.cr.KeyFile, + "-m", nf.Monitors, + } // FIXME: replace the ceph command with go-ceph API in future - cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "ls"} + cmd := []string{"tell", fmt.Sprintf("mds.%d", mdsRank), "client", "ls"} + cmd = append(cmd, arg...) stdout, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...) if err != nil { return nil, fmt.Errorf("failed to list active clients: %w, stderr: %q", err, stdErr) @@ -161,9 +167,15 @@ func listActiveClients(ctx context.Context) ([]activeClient, error) { return activeClients, nil } -func evictCephFSClient(ctx context.Context, clientID int) error { +func (nf *NetworkFence) evictCephFSClient(ctx context.Context, clientID int) error { + arg := []string{ + "--id", nf.cr.ID, + "--keyfile=" + nf.cr.KeyFile, + "-m", nf.Monitors, + } // FIXME: replace the ceph command with go-ceph API in future - cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)} + cmd := []string{"tell", fmt.Sprintf("mds.%d", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)} + cmd = append(cmd, arg...) _, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...) if err != nil { return fmt.Errorf("failed to evict client %d: %w, stderr: %q", clientID, err, stdErr) @@ -233,7 +245,7 @@ func (ac *activeClient) fetchID() (int, error) { func (nf *NetworkFence) AddClientEviction(ctx context.Context) error { evictedIPs := make(map[string]bool) // fetch active clients - activeClients, err := listActiveClients(ctx) + activeClients, err := nf.listActiveClients(ctx) if err != nil { return err } @@ -251,7 +263,7 @@ func (nf *NetworkFence) AddClientEviction(ctx context.Context) error { return fmt.Errorf("error fetching client ID: %w", err) } // evict the client - err = evictCephFSClient(ctx, clientID) + err = nf.evictCephFSClient(ctx, clientID) if err != nil { return fmt.Errorf("error evicting client %d: %w", clientID, err) } @@ -353,6 +365,13 @@ func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string, useR // RemoveNetworkFence unblocks access for all the IPs in the IP range mentioned via the CIDR block // using a network fence. +// Unfencing one of the protocols(CephFS or RBD) suggests the node is expected to be recovered, so +// both CephFS and RBD are expected to work again too. +// example: +// Create RBD NetworkFence CR for one IP 10.10.10.10 +// Created CephFS NetworkFence CR for IP range but above IP comes in the Range +// Delete the CephFS Network Fence CR to unblocklist the IP +// So now the IP (10.10.10.10) is (un)blocklisted and can be used by both protocols. func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error { hasBlocklistRangeSupport := true // for each CIDR block, convert it into a range of IPs so as to undo blocklisting operation. From 74dfb7b9138522d8418f62207c890b44b727125a Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Fri, 29 Sep 2023 11:32:01 +0530 Subject: [PATCH 7/8] cephfs: adds unit testing for unfencing procedure Signed-off-by: Riya Singhal --- internal/csi-addons/cephfs/network_fence_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/csi-addons/cephfs/network_fence_test.go b/internal/csi-addons/cephfs/network_fence_test.go index ce463f134..0d5cb9611 100644 --- a/internal/csi-addons/cephfs/network_fence_test.go +++ b/internal/csi-addons/cephfs/network_fence_test.go @@ -42,3 +42,19 @@ func TestFenceClusterNetwork(t *testing.T) { _, err := controller.FenceClusterNetwork(context.TODO(), req) assert.Error(t, err) } + +// TestUnfenceClusterNetwork is a minimal test for the UnfenceClusterNetwork() +// procedure. During unit-testing, there is no Ceph cluster available, so actual +// operations can not be performed. +func TestUnfenceClusterNetwork(t *testing.T) { + t.Parallel() + controller := NewFenceControllerServer() + + req := &fence.UnfenceClusterNetworkRequest{ + Parameters: map[string]string{}, + Secrets: nil, + Cidrs: nil, + } + _, err := controller.UnfenceClusterNetwork(context.TODO(), req) + assert.Error(t, err) +} From 1fc96783429623177e739281cbefd3f22406d833 Mon Sep 17 00:00:00 2001 From: Riya Singhal Date: Thu, 5 Oct 2023 21:01:58 +0530 Subject: [PATCH 8/8] cephfs: expose FenceControllerServer in driver.go to handle the requests Signed-off-by: Riya Singhal --- internal/cephfs/driver.go | 5 +++++ internal/csi-addons/cephfs/network_fence_test.go | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/cephfs/driver.go b/internal/cephfs/driver.go index 7c6f1b187..d134fc048 100644 --- a/internal/cephfs/driver.go +++ b/internal/cephfs/driver.go @@ -197,6 +197,11 @@ func (fs *Driver) setupCSIAddonsServer(conf *util.Config) error { is := casceph.NewIdentityServer(conf) fs.cas.RegisterService(is) + if conf.IsControllerServer { + fcs := casceph.NewFenceControllerServer() + fs.cas.RegisterService(fcs) + } + // start the server, this does not block, it runs a new go-routine err = fs.cas.Start() if err != nil { diff --git a/internal/csi-addons/cephfs/network_fence_test.go b/internal/csi-addons/cephfs/network_fence_test.go index 0d5cb9611..e2e97891a 100644 --- a/internal/csi-addons/cephfs/network_fence_test.go +++ b/internal/csi-addons/cephfs/network_fence_test.go @@ -20,9 +20,8 @@ import ( "context" "testing" - "github.com/stretchr/testify/assert" - "github.com/csi-addons/spec/lib/go/fence" + "github.com/stretchr/testify/assert" ) // TestFenceClusterNetwork is a minimal test for the FenceClusterNetwork()