cephfs: adds the implementation of client eviction

this commit adds client eviction to cephfs, based
on the IPs in cidr block, it evicts those IPs from
the network.

Signed-off-by: Riya Singhal <rsinghal@redhat.com>

internal/csi-addons/cephfs/network_fence.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2023 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cephfs
+
+import (
+	"context"
+	"errors"
+
+	nf "github.com/ceph/ceph-csi/internal/csi-addons/networkfence"
+	"github.com/ceph/ceph-csi/internal/util"
+
+	"github.com/csi-addons/spec/lib/go/fence"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// FenceControllerServer struct of cephFS CSI driver with supported methods
+// of CSI-Addons networkfence controller service spec.
+type FenceControllerServer struct {
+	*fence.UnimplementedFenceControllerServer
+}
+
+// NewFenceControllerServer creates a new FenceControllerServer which handles
+// the FenceController Service requests from the CSI-Addons specification.
+func NewFenceControllerServer() *FenceControllerServer {
+	return &FenceControllerServer{}
+}
+
+// RegisterService registers the FenceControllerServer's service
+// with the gRPC server.
+func (fcs *FenceControllerServer) RegisterService(server grpc.ServiceRegistrar) {
+	fence.RegisterFenceControllerServer(server, fcs)
+}
+
+// validateFenceClusterNetworkReq checks the sanity of FenceClusterNetworkRequest.
+func validateNetworkFenceReq(fenceClients []*fence.CIDR, options map[string]string) error {
+	if len(fenceClients) == 0 {
+		return errors.New("CIDR block cannot be empty")
+	}
+
+	if value, ok := options["clusterID"]; !ok || value == "" {
+		return errors.New("missing or empty clusterID")
+	}
+
+	return nil
+}
+
+// FenceClusterNetwork blocks access to a CIDR block by creating a network fence.
+// It evicts the IP addresses of clients, which are in CIDR block.
+func (fcs *FenceControllerServer) FenceClusterNetwork(
+	ctx context.Context,
+	req *fence.FenceClusterNetworkRequest,
+) (*fence.FenceClusterNetworkResponse, error) {
+	err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters)
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+
+	cr, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.InvalidArgument, err.Error())
+	}
+	defer cr.DeleteCredentials()
+
+	nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+
+	err = nwFence.AddClientEviction(ctx)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to fence CIDR block %q: %s", nwFence.Cidr, err.Error())
+	}
+
+	return &fence.FenceClusterNetworkResponse{}, nil
+}

internal/csi-addons/networkfence/fencing.go

@@ -1,9 +1,12 @@
 /*
-Copyright 2022 The Ceph-CSI Authors.
+Copyright 2023 The Ceph-CSI Authors.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -15,10 +18,13 @@ package networkfence
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/ceph/ceph-csi/internal/util"
 	"github.com/ceph/ceph-csi/internal/util/log"
@@ -29,6 +35,8 @@ import (
 const (
 	blocklistTime     = "157784760"
 	invalidCommandStr = "invalid command"
+	// we can always use mds rank 0, since all the clients have a session with rank-0.
+	mdsRank = "0"
 )
 
 // NetworkFence contains the CIDR blocks to be blocked.
@@ -38,6 +46,11 @@ type NetworkFence struct {
 	cr       *util.Credentials
 }
 
+// activeClient represents the structure of an active client.
+type activeClient struct {
+	Inst string `json:"inst"`
+}
+
 // NewNetworkFence returns a networkFence struct object from the Network fence/unfence request.
 func NewNetworkFence(
 	ctx context.Context,
@@ -132,6 +145,148 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error {
 	return nil
 }
 
+func listActiveClients(ctx context.Context) ([]activeClient, error) {
+	// FIXME: replace the ceph command with go-ceph API in future
+	cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "ls"}
+	stdout, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list active clients: %w, stderr: %q", err, stdErr)
+	}
+
+	var activeClients []activeClient
+	if err := json.Unmarshal([]byte(stdout), &activeClients); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+	}
+
+	return activeClients, nil
+}
+
+func evictCephFSClient(ctx context.Context, clientID int) error {
+	// FIXME: replace the ceph command with go-ceph API in future
+	cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)}
+	_, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
+	if err != nil {
+		return fmt.Errorf("failed to evict client %d: %w, stderr: %q", clientID, err, stdErr)
+	}
+	log.DebugLog(ctx, "client %s has been evicted from CephFS\n", clientID)
+
+	return nil
+}
+
+func isIPInCIDR(ctx context.Context, ip, cidr string) bool {
+	// Parse the CIDR block
+	_, ipCidr, err := net.ParseCIDR(cidr)
+	if err != nil {
+		log.ErrorLog(ctx, "error parsing CIDR block %s: %w\n", cidr, err)
+
+		return false
+	}
+
+	// Parse the IP address
+	ipAddress := net.ParseIP(ip)
+	if ipAddress == nil {
+		log.ErrorLog(ctx, "error parsing IP address %s\n", ip)
+
+		return false
+	}
+
+	// Check if the IP address is within the CIDR block
+	return ipCidr.Contains(ipAddress)
+}
+
+func (ac *activeClient) fetchIP() (string, error) {
+	// example: "inst": "client.4305 172.21.9.34:0/422650892",
+	// then returning value will be 172.21.9.34
+	clientInfo := ac.Inst
+	parts := strings.Fields(clientInfo)
+	if len(parts) >= 2 {
+		ip := strings.Split(parts[1], ":")[0]
+
+		return ip, nil
+	}
+
+	return "", fmt.Errorf("failed to extract IP address, incorrect format: %s", clientInfo)
+}
+
+func (ac *activeClient) fetchID() (int, error) {
+	// example: "inst": "client.4305 172.21.9.34:0/422650892",
+	// then returning value will be 4305
+	clientInfo := ac.Inst
+	parts := strings.Fields(clientInfo)
+	if len(parts) >= 1 {
+		clientIDStr := strings.TrimPrefix(parts[0], "client.")
+		clientID, err := strconv.Atoi(clientIDStr)
+		if err != nil {
+			return 0, fmt.Errorf("failed to convert client ID to int: %w", err)
+		}
+
+		return clientID, nil
+	}
+
+	return 0, fmt.Errorf("failed to extract client ID, incorrect format: %s", clientInfo)
+
+}
+
+// AddClientEviction blocks access for all the IPs in the CIDR block
+// using client eviction.
+// blocks the active clients listed in cidr, and the IPs
+// for whom there is no active client present too.
+func (nf *NetworkFence) AddClientEviction(ctx context.Context) error {
+	evictedIPs := make(map[string]bool)
+	// fetch active clients
+	activeClients, err := listActiveClients(ctx)
+	if err != nil {
+		return err
+	}
+	// iterate through CIDR blocks and check if any active client matches
+	for _, cidr := range nf.Cidr {
+		for _, client := range activeClients {
+			clientIP, err := client.fetchIP()
+			if err != nil {
+				return fmt.Errorf("error fetching client IP: %w", err)
+			}
+			// check if the clientIP is in the CIDR block
+			if isIPInCIDR(ctx, clientIP, cidr) {
+				clientID, err := client.fetchID()
+				if err != nil {
+					return fmt.Errorf("error fetching client ID: %w", err)
+				}
+				// evict the client
+				err = evictCephFSClient(ctx, clientID)
+				if err != nil {
+					return fmt.Errorf("error evicting client %d: %w", clientID, err)
+				}
+				log.DebugLog(ctx, "client %d has been evicted\n", clientID)
+				// add the CIDR to the list of blocklisted IPs
+				evictedIPs[clientIP] = true
+			}
+		}
+	}
+
+	// blocklist the IPs in CIDR without any active clients
+	for _, cidr := range nf.Cidr {
+		// check if the CIDR is evicted
+		// fetch the list of IPs from a CIDR block
+		hosts, err := getIPRange(cidr)
+		if err != nil {
+			return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range: %w", cidr, err)
+		}
+
+		// add ceph blocklist for each IP in the range mentioned by the CIDR
+		for _, host := range hosts {
+			if evictedIPs[host] {
+				continue
+			}
+			err = nf.addCephBlocklist(ctx, host, false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
 // getIPRange returns a list of IPs from the IP range
 // corresponding to a CIDR block.
 func getIPRange(cidr string) ([]string, error) {

internal/csi-addons/networkfence/fencing_test.go

@@ -1,9 +1,12 @@
 /*
-Copyright 2022 The Ceph-CSI Authors.
+Copyright 2023 The Ceph-CSI Authors.
+
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
+
     http://www.apache.org/licenses/LICENSE-2.0
+
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.