mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-09 16:00:22 +00:00
cephfs: adds the implementation of client eviction
this commit adds client eviction to cephfs, based on the IPs in cidr block, it evicts those IPs from the network. Signed-off-by: Riya Singhal <rsinghal@redhat.com>
This commit is contained in:
parent
b062479012
commit
9d5ee285a2
91
internal/csi-addons/cephfs/network_fence.go
Normal file
91
internal/csi-addons/cephfs/network_fence.go
Normal file
@ -0,0 +1,91 @@
|
||||
/*
|
||||
Copyright 2023 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
nf "github.com/ceph/ceph-csi/internal/csi-addons/networkfence"
|
||||
"github.com/ceph/ceph-csi/internal/util"
|
||||
|
||||
"github.com/csi-addons/spec/lib/go/fence"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// FenceControllerServer struct of cephFS CSI driver with supported methods
|
||||
// of CSI-Addons networkfence controller service spec.
|
||||
type FenceControllerServer struct {
|
||||
*fence.UnimplementedFenceControllerServer
|
||||
}
|
||||
|
||||
// NewFenceControllerServer creates a new FenceControllerServer which handles
|
||||
// the FenceController Service requests from the CSI-Addons specification.
|
||||
func NewFenceControllerServer() *FenceControllerServer {
|
||||
return &FenceControllerServer{}
|
||||
}
|
||||
|
||||
// RegisterService registers the FenceControllerServer's service
|
||||
// with the gRPC server.
|
||||
func (fcs *FenceControllerServer) RegisterService(server grpc.ServiceRegistrar) {
|
||||
fence.RegisterFenceControllerServer(server, fcs)
|
||||
}
|
||||
|
||||
// validateFenceClusterNetworkReq checks the sanity of FenceClusterNetworkRequest.
|
||||
func validateNetworkFenceReq(fenceClients []*fence.CIDR, options map[string]string) error {
|
||||
if len(fenceClients) == 0 {
|
||||
return errors.New("CIDR block cannot be empty")
|
||||
}
|
||||
|
||||
if value, ok := options["clusterID"]; !ok || value == "" {
|
||||
return errors.New("missing or empty clusterID")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FenceClusterNetwork blocks access to a CIDR block by creating a network fence.
|
||||
// It evicts the IP addresses of clients, which are in CIDR block.
|
||||
func (fcs *FenceControllerServer) FenceClusterNetwork(
|
||||
ctx context.Context,
|
||||
req *fence.FenceClusterNetworkRequest,
|
||||
) (*fence.FenceClusterNetworkResponse, error) {
|
||||
err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
cr, err := util.NewUserCredentials(req.GetSecrets())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
defer cr.DeleteCredentials()
|
||||
|
||||
nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
err = nwFence.AddClientEviction(ctx)
|
||||
if err != nil {
|
||||
return nil, status.Errorf(codes.Internal, "failed to fence CIDR block %q: %s", nwFence.Cidr, err.Error())
|
||||
}
|
||||
|
||||
return &fence.FenceClusterNetworkResponse{}, nil
|
||||
}
|
@ -1,9 +1,12 @@
|
||||
/*
|
||||
Copyright 2022 The Ceph-CSI Authors.
|
||||
Copyright 2023 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
@ -15,10 +18,13 @@ package networkfence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ceph/ceph-csi/internal/util"
|
||||
"github.com/ceph/ceph-csi/internal/util/log"
|
||||
@ -29,6 +35,8 @@ import (
|
||||
const (
|
||||
blocklistTime = "157784760"
|
||||
invalidCommandStr = "invalid command"
|
||||
// we can always use mds rank 0, since all the clients have a session with rank-0.
|
||||
mdsRank = "0"
|
||||
)
|
||||
|
||||
// NetworkFence contains the CIDR blocks to be blocked.
|
||||
@ -38,6 +46,11 @@ type NetworkFence struct {
|
||||
cr *util.Credentials
|
||||
}
|
||||
|
||||
// activeClient represents the structure of an active client.
|
||||
type activeClient struct {
|
||||
Inst string `json:"inst"`
|
||||
}
|
||||
|
||||
// NewNetworkFence returns a networkFence struct object from the Network fence/unfence request.
|
||||
func NewNetworkFence(
|
||||
ctx context.Context,
|
||||
@ -132,6 +145,148 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func listActiveClients(ctx context.Context) ([]activeClient, error) {
|
||||
// FIXME: replace the ceph command with go-ceph API in future
|
||||
cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "ls"}
|
||||
stdout, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list active clients: %w, stderr: %q", err, stdErr)
|
||||
}
|
||||
|
||||
var activeClients []activeClient
|
||||
if err := json.Unmarshal([]byte(stdout), &activeClients); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
|
||||
}
|
||||
|
||||
return activeClients, nil
|
||||
}
|
||||
|
||||
func evictCephFSClient(ctx context.Context, clientID int) error {
|
||||
// FIXME: replace the ceph command with go-ceph API in future
|
||||
cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)}
|
||||
_, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to evict client %d: %w, stderr: %q", clientID, err, stdErr)
|
||||
}
|
||||
log.DebugLog(ctx, "client %s has been evicted from CephFS\n", clientID)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func isIPInCIDR(ctx context.Context, ip, cidr string) bool {
|
||||
// Parse the CIDR block
|
||||
_, ipCidr, err := net.ParseCIDR(cidr)
|
||||
if err != nil {
|
||||
log.ErrorLog(ctx, "error parsing CIDR block %s: %w\n", cidr, err)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse the IP address
|
||||
ipAddress := net.ParseIP(ip)
|
||||
if ipAddress == nil {
|
||||
log.ErrorLog(ctx, "error parsing IP address %s\n", ip)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if the IP address is within the CIDR block
|
||||
return ipCidr.Contains(ipAddress)
|
||||
}
|
||||
|
||||
func (ac *activeClient) fetchIP() (string, error) {
|
||||
// example: "inst": "client.4305 172.21.9.34:0/422650892",
|
||||
// then returning value will be 172.21.9.34
|
||||
clientInfo := ac.Inst
|
||||
parts := strings.Fields(clientInfo)
|
||||
if len(parts) >= 2 {
|
||||
ip := strings.Split(parts[1], ":")[0]
|
||||
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("failed to extract IP address, incorrect format: %s", clientInfo)
|
||||
}
|
||||
|
||||
func (ac *activeClient) fetchID() (int, error) {
|
||||
// example: "inst": "client.4305 172.21.9.34:0/422650892",
|
||||
// then returning value will be 4305
|
||||
clientInfo := ac.Inst
|
||||
parts := strings.Fields(clientInfo)
|
||||
if len(parts) >= 1 {
|
||||
clientIDStr := strings.TrimPrefix(parts[0], "client.")
|
||||
clientID, err := strconv.Atoi(clientIDStr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to convert client ID to int: %w", err)
|
||||
}
|
||||
|
||||
return clientID, nil
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("failed to extract client ID, incorrect format: %s", clientInfo)
|
||||
|
||||
}
|
||||
|
||||
// AddClientEviction blocks access for all the IPs in the CIDR block
|
||||
// using client eviction.
|
||||
// blocks the active clients listed in cidr, and the IPs
|
||||
// for whom there is no active client present too.
|
||||
func (nf *NetworkFence) AddClientEviction(ctx context.Context) error {
|
||||
evictedIPs := make(map[string]bool)
|
||||
// fetch active clients
|
||||
activeClients, err := listActiveClients(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// iterate through CIDR blocks and check if any active client matches
|
||||
for _, cidr := range nf.Cidr {
|
||||
for _, client := range activeClients {
|
||||
clientIP, err := client.fetchIP()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching client IP: %w", err)
|
||||
}
|
||||
// check if the clientIP is in the CIDR block
|
||||
if isIPInCIDR(ctx, clientIP, cidr) {
|
||||
clientID, err := client.fetchID()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error fetching client ID: %w", err)
|
||||
}
|
||||
// evict the client
|
||||
err = evictCephFSClient(ctx, clientID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error evicting client %d: %w", clientID, err)
|
||||
}
|
||||
log.DebugLog(ctx, "client %d has been evicted\n", clientID)
|
||||
// add the CIDR to the list of blocklisted IPs
|
||||
evictedIPs[clientIP] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// blocklist the IPs in CIDR without any active clients
|
||||
for _, cidr := range nf.Cidr {
|
||||
// check if the CIDR is evicted
|
||||
// fetch the list of IPs from a CIDR block
|
||||
hosts, err := getIPRange(cidr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range: %w", cidr, err)
|
||||
}
|
||||
|
||||
// add ceph blocklist for each IP in the range mentioned by the CIDR
|
||||
for _, host := range hosts {
|
||||
if evictedIPs[host] {
|
||||
continue
|
||||
}
|
||||
err = nf.addCephBlocklist(ctx, host, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getIPRange returns a list of IPs from the IP range
|
||||
// corresponding to a CIDR block.
|
||||
func getIPRange(cidr string) ([]string, error) {
|
||||
|
@ -1,9 +1,12 @@
|
||||
/*
|
||||
Copyright 2022 The Ceph-CSI Authors.
|
||||
Copyright 2023 The Ceph-CSI Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
|
Loading…
Reference in New Issue
Block a user