mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-09 16:00:22 +00:00
cephfs: adds the implementation of client eviction
this commit adds client eviction to cephfs, based on the IPs in cidr block, it evicts those IPs from the network. Signed-off-by: Riya Singhal <rsinghal@redhat.com>
This commit is contained in:
parent
b062479012
commit
9d5ee285a2
91
internal/csi-addons/cephfs/network_fence.go
Normal file
91
internal/csi-addons/cephfs/network_fence.go
Normal file
@ -0,0 +1,91 @@
|
|||||||
|
/*
|
||||||
|
Copyright 2023 The Ceph-CSI Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package cephfs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
nf "github.com/ceph/ceph-csi/internal/csi-addons/networkfence"
|
||||||
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
|
|
||||||
|
"github.com/csi-addons/spec/lib/go/fence"
|
||||||
|
"google.golang.org/grpc"
|
||||||
|
"google.golang.org/grpc/codes"
|
||||||
|
"google.golang.org/grpc/status"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FenceControllerServer struct of cephFS CSI driver with supported methods
|
||||||
|
// of CSI-Addons networkfence controller service spec.
|
||||||
|
type FenceControllerServer struct {
|
||||||
|
*fence.UnimplementedFenceControllerServer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewFenceControllerServer creates a new FenceControllerServer which handles
|
||||||
|
// the FenceController Service requests from the CSI-Addons specification.
|
||||||
|
func NewFenceControllerServer() *FenceControllerServer {
|
||||||
|
return &FenceControllerServer{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterService registers the FenceControllerServer's service
|
||||||
|
// with the gRPC server.
|
||||||
|
func (fcs *FenceControllerServer) RegisterService(server grpc.ServiceRegistrar) {
|
||||||
|
fence.RegisterFenceControllerServer(server, fcs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateFenceClusterNetworkReq checks the sanity of FenceClusterNetworkRequest.
|
||||||
|
func validateNetworkFenceReq(fenceClients []*fence.CIDR, options map[string]string) error {
|
||||||
|
if len(fenceClients) == 0 {
|
||||||
|
return errors.New("CIDR block cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
if value, ok := options["clusterID"]; !ok || value == "" {
|
||||||
|
return errors.New("missing or empty clusterID")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FenceClusterNetwork blocks access to a CIDR block by creating a network fence.
|
||||||
|
// It evicts the IP addresses of clients, which are in CIDR block.
|
||||||
|
func (fcs *FenceControllerServer) FenceClusterNetwork(
|
||||||
|
ctx context.Context,
|
||||||
|
req *fence.FenceClusterNetworkRequest,
|
||||||
|
) (*fence.FenceClusterNetworkResponse, error) {
|
||||||
|
err := validateNetworkFenceReq(req.GetCidrs(), req.Parameters)
|
||||||
|
if err != nil {
|
||||||
|
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
cr, err := util.NewUserCredentials(req.GetSecrets())
|
||||||
|
if err != nil {
|
||||||
|
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||||
|
}
|
||||||
|
defer cr.DeleteCredentials()
|
||||||
|
|
||||||
|
nwFence, err := nf.NewNetworkFence(ctx, cr, req.Cidrs, req.GetParameters())
|
||||||
|
if err != nil {
|
||||||
|
return nil, status.Error(codes.Internal, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
err = nwFence.AddClientEviction(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, status.Errorf(codes.Internal, "failed to fence CIDR block %q: %s", nwFence.Cidr, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
return &fence.FenceClusterNetworkResponse{}, nil
|
||||||
|
}
|
@ -1,9 +1,12 @@
|
|||||||
/*
|
/*
|
||||||
Copyright 2022 The Ceph-CSI Authors.
|
Copyright 2023 The Ceph-CSI Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
You may obtain a copy of the License at
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
Unless required by applicable law or agreed to in writing, software
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
@ -15,10 +18,13 @@ package networkfence
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/ceph/ceph-csi/internal/util"
|
"github.com/ceph/ceph-csi/internal/util"
|
||||||
"github.com/ceph/ceph-csi/internal/util/log"
|
"github.com/ceph/ceph-csi/internal/util/log"
|
||||||
@ -29,6 +35,8 @@ import (
|
|||||||
const (
|
const (
|
||||||
blocklistTime = "157784760"
|
blocklistTime = "157784760"
|
||||||
invalidCommandStr = "invalid command"
|
invalidCommandStr = "invalid command"
|
||||||
|
// we can always use mds rank 0, since all the clients have a session with rank-0.
|
||||||
|
mdsRank = "0"
|
||||||
)
|
)
|
||||||
|
|
||||||
// NetworkFence contains the CIDR blocks to be blocked.
|
// NetworkFence contains the CIDR blocks to be blocked.
|
||||||
@ -38,6 +46,11 @@ type NetworkFence struct {
|
|||||||
cr *util.Credentials
|
cr *util.Credentials
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// activeClient represents the structure of an active client.
|
||||||
|
type activeClient struct {
|
||||||
|
Inst string `json:"inst"`
|
||||||
|
}
|
||||||
|
|
||||||
// NewNetworkFence returns a networkFence struct object from the Network fence/unfence request.
|
// NewNetworkFence returns a networkFence struct object from the Network fence/unfence request.
|
||||||
func NewNetworkFence(
|
func NewNetworkFence(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@ -132,6 +145,148 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func listActiveClients(ctx context.Context) ([]activeClient, error) {
|
||||||
|
// FIXME: replace the ceph command with go-ceph API in future
|
||||||
|
cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "ls"}
|
||||||
|
stdout, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to list active clients: %w, stderr: %q", err, stdErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
var activeClients []activeClient
|
||||||
|
if err := json.Unmarshal([]byte(stdout), &activeClients); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return activeClients, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func evictCephFSClient(ctx context.Context, clientID int) error {
|
||||||
|
// FIXME: replace the ceph command with go-ceph API in future
|
||||||
|
cmd := []string{"tell", fmt.Sprintf("mds.%s", mdsRank), "client", "evict", fmt.Sprintf("id=%d", clientID)}
|
||||||
|
_, stdErr, err := util.ExecCommandWithTimeout(ctx, 2*time.Minute, "ceph", cmd...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to evict client %d: %w, stderr: %q", clientID, err, stdErr)
|
||||||
|
}
|
||||||
|
log.DebugLog(ctx, "client %s has been evicted from CephFS\n", clientID)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isIPInCIDR(ctx context.Context, ip, cidr string) bool {
|
||||||
|
// Parse the CIDR block
|
||||||
|
_, ipCidr, err := net.ParseCIDR(cidr)
|
||||||
|
if err != nil {
|
||||||
|
log.ErrorLog(ctx, "error parsing CIDR block %s: %w\n", cidr, err)
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the IP address
|
||||||
|
ipAddress := net.ParseIP(ip)
|
||||||
|
if ipAddress == nil {
|
||||||
|
log.ErrorLog(ctx, "error parsing IP address %s\n", ip)
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the IP address is within the CIDR block
|
||||||
|
return ipCidr.Contains(ipAddress)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *activeClient) fetchIP() (string, error) {
|
||||||
|
// example: "inst": "client.4305 172.21.9.34:0/422650892",
|
||||||
|
// then returning value will be 172.21.9.34
|
||||||
|
clientInfo := ac.Inst
|
||||||
|
parts := strings.Fields(clientInfo)
|
||||||
|
if len(parts) >= 2 {
|
||||||
|
ip := strings.Split(parts[1], ":")[0]
|
||||||
|
|
||||||
|
return ip, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("failed to extract IP address, incorrect format: %s", clientInfo)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ac *activeClient) fetchID() (int, error) {
|
||||||
|
// example: "inst": "client.4305 172.21.9.34:0/422650892",
|
||||||
|
// then returning value will be 4305
|
||||||
|
clientInfo := ac.Inst
|
||||||
|
parts := strings.Fields(clientInfo)
|
||||||
|
if len(parts) >= 1 {
|
||||||
|
clientIDStr := strings.TrimPrefix(parts[0], "client.")
|
||||||
|
clientID, err := strconv.Atoi(clientIDStr)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to convert client ID to int: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return clientID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0, fmt.Errorf("failed to extract client ID, incorrect format: %s", clientInfo)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddClientEviction blocks access for all the IPs in the CIDR block
|
||||||
|
// using client eviction.
|
||||||
|
// blocks the active clients listed in cidr, and the IPs
|
||||||
|
// for whom there is no active client present too.
|
||||||
|
func (nf *NetworkFence) AddClientEviction(ctx context.Context) error {
|
||||||
|
evictedIPs := make(map[string]bool)
|
||||||
|
// fetch active clients
|
||||||
|
activeClients, err := listActiveClients(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// iterate through CIDR blocks and check if any active client matches
|
||||||
|
for _, cidr := range nf.Cidr {
|
||||||
|
for _, client := range activeClients {
|
||||||
|
clientIP, err := client.fetchIP()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error fetching client IP: %w", err)
|
||||||
|
}
|
||||||
|
// check if the clientIP is in the CIDR block
|
||||||
|
if isIPInCIDR(ctx, clientIP, cidr) {
|
||||||
|
clientID, err := client.fetchID()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error fetching client ID: %w", err)
|
||||||
|
}
|
||||||
|
// evict the client
|
||||||
|
err = evictCephFSClient(ctx, clientID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error evicting client %d: %w", clientID, err)
|
||||||
|
}
|
||||||
|
log.DebugLog(ctx, "client %d has been evicted\n", clientID)
|
||||||
|
// add the CIDR to the list of blocklisted IPs
|
||||||
|
evictedIPs[clientIP] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// blocklist the IPs in CIDR without any active clients
|
||||||
|
for _, cidr := range nf.Cidr {
|
||||||
|
// check if the CIDR is evicted
|
||||||
|
// fetch the list of IPs from a CIDR block
|
||||||
|
hosts, err := getIPRange(cidr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to convert CIDR block %s to corresponding IP range: %w", cidr, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// add ceph blocklist for each IP in the range mentioned by the CIDR
|
||||||
|
for _, host := range hosts {
|
||||||
|
if evictedIPs[host] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
err = nf.addCephBlocklist(ctx, host, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// getIPRange returns a list of IPs from the IP range
|
// getIPRange returns a list of IPs from the IP range
|
||||||
// corresponding to a CIDR block.
|
// corresponding to a CIDR block.
|
||||||
func getIPRange(cidr string) ([]string, error) {
|
func getIPRange(cidr string) ([]string, error) {
|
||||||
|
@ -1,9 +1,12 @@
|
|||||||
/*
|
/*
|
||||||
Copyright 2022 The Ceph-CSI Authors.
|
Copyright 2023 The Ceph-CSI Authors.
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
You may obtain a copy of the License at
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
http://www.apache.org/licenses/LICENSE-2.0
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
Unless required by applicable law or agreed to in writing, software
|
Unless required by applicable law or agreed to in writing, software
|
||||||
distributed under the License is distributed on an "AS IS" BASIS,
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
Loading…
Reference in New Issue
Block a user