From 57a4715e95d998399c60235344a8962ea7023d78 Mon Sep 17 00:00:00 2001 From: Huamin Chen Date: Thu, 18 Jan 2018 19:13:08 +0000 Subject: [PATCH] rbd plugin refactoring: remove k8s clientSet Signed-off-by: Huamin Chen --- pkg/rbd/controllerserver.go | 59 ++++++------------ pkg/rbd/nodeserver.go | 24 +------- pkg/rbd/rbd.go | 15 ++--- pkg/rbd/rbd_util.go | 119 ++++++++++-------------------------- rbd/main.go | 21 ++----- 5 files changed, 65 insertions(+), 173 deletions(-) diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index a80f98659..4dca244d6 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -24,8 +24,6 @@ import ( "github.com/pborman/uuid" "golang.org/x/net/context" - "k8s.io/client-go/kubernetes" - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/drivers/pkg/csi-common" ) @@ -36,7 +34,6 @@ const ( type controllerServer struct { *csicommon.DefaultControllerServer - clientSet *kubernetes.Clientset } func GetVersionString(ver *csi.Version) string { @@ -49,7 +46,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - volOptions, err := getRBDVolumeOptions(req.Parameters, cs.clientSet) + volOptions, err := getRBDVolumeOptions(req.GetParameters()) if err != nil { return nil, err } @@ -101,6 +98,25 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return nil, err } + // For now the image get unconditionally deleted, but here retention policy can be checked + volumeID := req.GetVolumeId() + volOptions := &rbdVolumeOptions{} + if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil { + return nil, err + } + + volName := volOptions.VolName + // Deleting rbd image + glog.V(4).Infof("deleting volume %s", volName) + if err := deleteRBDImage(volOptions); err != nil { + glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err) + return nil, err + } + // Removing persistent storage file for the unmapped volume + if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil { + return nil, err + } + return &csi.DeleteVolumeResponse{}, nil } @@ -114,44 +130,9 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req } func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { - - // For now the image get unconditionally deleted, but here retention policy can be checked - volumeID := req.GetVolumeId() - volOptions := &rbdVolumeOptions{} - if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil { - return nil, err - } - - volName := volOptions.VolName - // Recover rbd secret key value, for now by k8s specific call - id := volOptions.AdminID - secretName := volOptions.AdminSecretName - secretNamespace := volOptions.AdminSecretNamespace - if id == "" { - secretName = volOptions.UserSecretName - secretNamespace = volOptions.UserSecretNamespace - } - if key, err := parseStorageClassSecret(secretName, secretNamespace, cs.clientSet); err != nil { - return nil, err - } else { - volOptions.adminSecret = key - } - - // Deleting rbd image - glog.V(4).Infof("deleting volume %s", volName) - if err := deleteRBDImage(volOptions); err != nil { - glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err) - return nil, err - } - // Removing persistent storage file for the unmapped volume - if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil { - return nil, err - } - return &csi.ControllerUnpublishVolumeResponse{}, nil } func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { - return &csi.ControllerPublishVolumeResponse{}, nil } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 88860e926..93a9b4e77 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -28,7 +28,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/util/mount" "github.com/kubernetes-csi/drivers/pkg/csi-common" @@ -36,7 +36,6 @@ import ( type nodeServer struct { *csicommon.DefaultNodeServer - clientSet *kubernetes.Clientset } func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { @@ -63,7 +62,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if !notMnt { return &csi.NodePublishVolumeResponse{}, nil } - volOptions, err := getRBDVolumeOptions(req.VolumeAttributes, ns.clientSet) + volOptions, err := getRBDVolumeOptions(req.VolumeAttributes) if err != nil { return nil, err } @@ -92,9 +91,6 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil { return nil, err } - // Storing rbd device path - - volOptions.ImageMapping = map[string]string{volOptions.VolName: devicePath} // Storing volInfo into a persistent file if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil { glog.Warningf("rbd: failed to store volInfo with error: %v", err) @@ -112,20 +108,6 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu } volName := volOptions.VolName - // Recover rbd secret key value, for now by k8s specific call - id := volOptions.AdminID - secretName := volOptions.AdminSecretName - secretNamespace := volOptions.AdminSecretNamespace - if id == "" { - secretName = volOptions.UserSecretName - secretNamespace = volOptions.UserSecretNamespace - } - if key, err := parseStorageClassSecret(secretName, secretNamespace, ns.clientSet); err != nil { - return nil, err - } else { - volOptions.adminSecret = key - } - notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -141,7 +123,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu // Unmapping rbd device glog.V(4).Infof("deleting volume %s", volName) if err := detachRBDImage(volOptions); err != nil { - glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.ImageMapping[volName], err) + glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.VolName, err) return nil, err } // Removing persistent storage file for the unmapped volume diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 2273cdedc..b8329d8f5 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -17,11 +17,10 @@ limitations under the License. package rbd import ( - "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/glog" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/kubernetes-csi/drivers/pkg/csi-common" - "k8s.io/client-go/kubernetes" ) // PluginFolder defines the location of rbdplugin @@ -61,21 +60,19 @@ func NewIdentityServer(d *csicommon.CSIDriver) *identityServer { } } -func NewControllerServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *controllerServer { +func NewControllerServer(d *csicommon.CSIDriver) *controllerServer { return &controllerServer{ DefaultControllerServer: csicommon.NewDefaultControllerServer(d), - clientSet: clientSet, } } -func NewNodeServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *nodeServer { +func NewNodeServer(d *csicommon.CSIDriver) *nodeServer { return &nodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(d), - clientSet: clientSet, } } -func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.Clientset) { +func (rbd *rbd) Run(driverName, nodeID, endpoint string) { glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version)) // Initialize default library driver @@ -91,8 +88,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.C // Create GRPC servers rbd.ids = NewIdentityServer(rbd.driver) - rbd.ns = NewNodeServer(rbd.driver, clientSet) - rbd.cs = NewControllerServer(rbd.driver, clientSet) + rbd.ns = NewNodeServer(rbd.driver) + rbd.cs = NewControllerServer(rbd.driver) s := csicommon.NewNonBlockingGRPCServer() s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns) s.Wait() diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 6ba6b4314..426d2b6d7 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -21,9 +21,7 @@ import ( "fmt" "github.com/golang/glog" "io/ioutil" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/pkg/util/keymutex" "os" "os/exec" @@ -47,20 +45,15 @@ const ( ) type rbdVolumeOptions struct { - VolName string `json:"volName"` - Monitors string `json:"monitors"` - Pool string `json:"pool"` - AdminSecretName string `json:"adminSecret"` - AdminSecretNamespace string `json:"adminSecretNamespace"` - AdminID string `json:"adminID"` - UserID string `json:"userID"` - UserSecretName string `json:"userSecret"` - UserSecretNamespace string `json:"userSecretNamespace"` - ImageFormat string `json:"imageFormat"` - ImageFeatures []string `json:"imageFeatures"` - ImageMapping map[string]string `json:"imageMapping"` - adminSecret string - userSecret string + VolName string `json:"volName"` + Monitors string `json:"monitors"` + Pool string `json:"pool"` + AdminID string `json:"adminID"` + AdminSecret string `json:"adminSecret"` + UserID string `json:"userID"` + UserSecret string `json:"userSecret"` + ImageFormat string `json:"imageFormat"` + ImageFeatures []string `json:"imageFeatures"` } var attachdetachMutex = keymutex.NewKeyMutex() @@ -76,11 +69,11 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error { volSzGB := fmt.Sprintf("%dG", volSz) if pOpts.ImageFormat == rbdImageFormat2 { - glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret) + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret) } else { - glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret) + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret) } - args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.adminSecret, "--image-format", pOpts.ImageFormat} + args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { // if no image features is provided, it results in empty string // which disable all RBD image format 2 features as we expected @@ -106,10 +99,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) { image := b.VolName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. id := b.AdminID - secret := b.adminSecret + secret := b.AdminSecret if id == "" { id = b.UserID - secret = b.userSecret + secret = b.UserSecret } glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) @@ -152,10 +145,10 @@ func deleteRBDImage(b *rbdVolumeOptions) error { return fmt.Errorf("rbd %s is still being used", image) } id := b.AdminID - secret := b.adminSecret + secret := b.AdminSecret if id == "" { id = b.UserID - secret = b.userSecret + secret = b.UserSecret } glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) @@ -173,25 +166,16 @@ func execCommand(command string, args []string) ([]byte, error) { return cmd.CombinedOutput() } -func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Clientset) (*rbdVolumeOptions, error) { +func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) { rbdVolume := &rbdVolumeOptions{} var ok bool - var err error - rbdVolume.AdminID, ok = volOptions["adminId"] + rbdVolume.AdminID, ok = volOptions["adminID"] if !ok { - return nil, fmt.Errorf("Missing required parameter adminId") + return nil, fmt.Errorf("Missing required parameter adminID") } - rbdVolume.AdminSecretName, ok = volOptions["adminSecretName"] + rbdVolume.AdminSecret, ok = volOptions["adminSecret"] if !ok { - return nil, fmt.Errorf("Missing required parameter adminSecretName") - } - rbdVolume.AdminSecretNamespace, ok = volOptions["adminSecretNamespace"] - if !ok { - rbdVolume.AdminSecretNamespace = "default" - } - rbdVolume.adminSecret, err = parseStorageClassSecret(rbdVolume.AdminSecretName, rbdVolume.AdminSecretNamespace, client) - if err != nil { - return nil, fmt.Errorf("Failed to retrieve Admin secret %v", err) + return nil, fmt.Errorf("Missing required parameter adminSecret") } rbdVolume.Pool, ok = volOptions["pool"] if !ok { @@ -201,23 +185,13 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client if !ok { return nil, fmt.Errorf("Missing required parameter monitors") } - if err != nil { - return nil, err - } - rbdVolume.UserID, ok = volOptions["userId"] + rbdVolume.UserID, ok = volOptions["userID"] if !ok { - return nil, fmt.Errorf("Missing required parameter userId") + return nil, fmt.Errorf("Missing required parameter userID") } - rbdVolume.UserSecretName, ok = volOptions["userSecretName"] - if ok { - rbdVolume.UserSecretNamespace, ok = volOptions["userSecretNamespace"] - if !ok { - rbdVolume.UserSecretNamespace = "default" - } - rbdVolume.userSecret, err = parseStorageClassSecret(rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, client) - if err != nil { - glog.Errorf("failed to retrieve user's secret: %s/%s (%v)", rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, err) - } + rbdVolume.UserSecret, ok = volOptions["userSecret"] + if !ok { + return nil, fmt.Errorf("Missing required parameter userSecret") } rbdVolume.ImageFormat, ok = volOptions["imageFormat"] if !ok { @@ -227,25 +201,6 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client return rbdVolume, nil } -func parseStorageClassSecret(secretName string, namespace string, client *kubernetes.Clientset) (string, error) { - if client == nil { - return "", fmt.Errorf("Cannot get kube client") - } - secrets, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) - if err != nil { - return "", err - } - secret := "" - for k, v := range secrets.Data { - if k == secretName { - return string(v), nil - } - secret = string(v) - } - - return secret, nil -} - func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) { var err error var output []byte @@ -283,13 +238,8 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) { } glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) - // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - id := volOptions.AdminID - secret := volOptions.adminSecret - if id == "" { - id = volOptions.UserID - secret = volOptions.userSecret - } + id := volOptions.UserID + secret := volOptions.UserSecret output, err = execCommand("rbd", []string{ "map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret}) @@ -311,17 +261,12 @@ func detachRBDImage(volOptions *rbdVolumeOptions) error { var output []byte image := volOptions.VolName - glog.V(1).Infof("rbd: unmap device %s", volOptions.ImageMapping[image]) - // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - id := volOptions.AdminID - secret := volOptions.adminSecret - if id == "" { - id = volOptions.UserID - secret = volOptions.userSecret - } + glog.V(1).Infof("rbd: unmap device %s", image) + id := volOptions.UserID + secret := volOptions.UserSecret output, err = execCommand("rbd", []string{ - "unmap", volOptions.ImageMapping[image], "--id", id, "--key=" + secret}) + "unmap", image, "--id", id, "--key=" + secret}) if err != nil { glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output)) return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output)) diff --git a/rbd/main.go b/rbd/main.go index bf327123f..6a2a12a40 100644 --- a/rbd/main.go +++ b/rbd/main.go @@ -18,13 +18,11 @@ package main import ( "flag" - "github.com/golang/glog" "os" "path" "github.com/ceph/ceph-csi/pkg/rbd" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" + "github.com/golang/glog" ) func init() { @@ -40,17 +38,6 @@ var ( func main() { flag.Parse() - // creates the in-cluster config - config, err := rest.InClusterConfig() - if err != nil { - panic(err.Error()) - } - // creates the clientset - clientSet, err := kubernetes.NewForConfig(config) - if err != nil { - panic(err.Error()) - } - if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil { glog.Errorf("failed to create persistent storage for controller %v", err) os.Exit(1) @@ -60,13 +47,13 @@ func main() { os.Exit(1) } - handle(clientSet) + handle() os.Exit(0) } -func handle(clientSet *kubernetes.Clientset) { +func handle() { driver := rbd.GetRBDDriver() - driver.Run(*driverName, *nodeID, *endpoint, clientSet) + driver.Run(*driverName, *nodeID, *endpoint) } func createPersistentStorage(persistentStoragePath string) error {