rbd plugin refactoring: remove k8s clientSet

Signed-off-by: Huamin Chen <hchen@redhat.com>

pkg/rbd/controllerserver.go

@@ -24,8 +24,6 @@ import (
 	"github.com/pborman/uuid"
 	"golang.org/x/net/context"
 
-	"k8s.io/client-go/kubernetes"
-
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 )
@@ -36,7 +34,6 @@ const (
 
 type controllerServer struct {
 	*csicommon.DefaultControllerServer
-	clientSet *kubernetes.Clientset
 }
 
 func GetVersionString(ver *csi.Version) string {
@@ -49,7 +46,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}
 
-	volOptions, err := getRBDVolumeOptions(req.Parameters, cs.clientSet)
+	volOptions, err := getRBDVolumeOptions(req.GetParameters())
 	if err != nil {
 		return nil, err
 	}
@@ -101,6 +98,25 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		return nil, err
 	}
 
+	// For now the image get unconditionally deleted, but here retention policy can be checked
+	volumeID := req.GetVolumeId()
+	volOptions := &rbdVolumeOptions{}
+	if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
+		return nil, err
+	}
+
+	volName := volOptions.VolName
+	// Deleting rbd image
+	glog.V(4).Infof("deleting volume %s", volName)
+	if err := deleteRBDImage(volOptions); err != nil {
+		glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
+		return nil, err
+	}
+	// Removing persistent storage file for the unmapped volume
+	if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
+		return nil, err
+	}
+
 	return &csi.DeleteVolumeResponse{}, nil
 }
 
@@ -114,44 +130,9 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
 }
 
 func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
-
-	// For now the image get unconditionally deleted, but here retention policy can be checked
-	volumeID := req.GetVolumeId()
-	volOptions := &rbdVolumeOptions{}
-	if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
-		return nil, err
-	}
-
-	volName := volOptions.VolName
-	// Recover rbd secret key value, for now by k8s specific call
-	id := volOptions.AdminID
-	secretName := volOptions.AdminSecretName
-	secretNamespace := volOptions.AdminSecretNamespace
-	if id == "" {
-		secretName = volOptions.UserSecretName
-		secretNamespace = volOptions.UserSecretNamespace
-	}
-	if key, err := parseStorageClassSecret(secretName, secretNamespace, cs.clientSet); err != nil {
-		return nil, err
-	} else {
-		volOptions.adminSecret = key
-	}
-
-	// Deleting rbd image
-	glog.V(4).Infof("deleting volume %s", volName)
-	if err := deleteRBDImage(volOptions); err != nil {
-		glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
-		return nil, err
-	}
-	// Removing persistent storage file for the unmapped volume
-	if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
-		return nil, err
-	}
-
 	return &csi.ControllerUnpublishVolumeResponse{}, nil
 }
 
 func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
-
 	return &csi.ControllerPublishVolumeResponse{}, nil
 }

pkg/rbd/nodeserver.go

@@ -28,7 +28,7 @@ import (
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
-	"k8s.io/client-go/kubernetes"
+
 	"k8s.io/kubernetes/pkg/util/mount"
 
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
@@ -36,7 +36,6 @@ import (
 
 type nodeServer struct {
 	*csicommon.DefaultNodeServer
-	clientSet *kubernetes.Clientset
 }
 
 func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
@@ -63,7 +62,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	if !notMnt {
 		return &csi.NodePublishVolumeResponse{}, nil
 	}
-	volOptions, err := getRBDVolumeOptions(req.VolumeAttributes, ns.clientSet)
+	volOptions, err := getRBDVolumeOptions(req.VolumeAttributes)
 	if err != nil {
 		return nil, err
 	}
@@ -92,9 +91,6 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
 		return nil, err
 	}
-	// Storing rbd device path
-
-	volOptions.ImageMapping = map[string]string{volOptions.VolName: devicePath}
 	// Storing volInfo into a persistent file
 	if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil {
 		glog.Warningf("rbd: failed to store volInfo with error: %v", err)
@@ -112,20 +108,6 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 	}
 	volName := volOptions.VolName
 
-	// Recover rbd secret key value, for now by k8s specific call
-	id := volOptions.AdminID
-	secretName := volOptions.AdminSecretName
-	secretNamespace := volOptions.AdminSecretNamespace
-	if id == "" {
-		secretName = volOptions.UserSecretName
-		secretNamespace = volOptions.UserSecretNamespace
-	}
-	if key, err := parseStorageClassSecret(secretName, secretNamespace, ns.clientSet); err != nil {
-		return nil, err
-	} else {
-		volOptions.adminSecret = key
-	}
-
 	notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
 	if err != nil {
 		return nil, status.Error(codes.Internal, err.Error())
@@ -141,7 +123,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 	// Unmapping rbd device
 	glog.V(4).Infof("deleting volume %s", volName)
 	if err := detachRBDImage(volOptions); err != nil {
-		glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.ImageMapping[volName], err)
+		glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.VolName, err)
 		return nil, err
 	}
 	// Removing persistent storage file for the unmapped volume

pkg/rbd/rbd.go

@@ -17,11 +17,10 @@ limitations under the License.
 package rbd
 
 import (
-	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/golang/glog"
 
+	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
-	"k8s.io/client-go/kubernetes"
 )
 
 // PluginFolder defines the location of rbdplugin
@@ -61,21 +60,19 @@ func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
 	}
 }
 
-func NewControllerServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *controllerServer {
+func NewControllerServer(d *csicommon.CSIDriver) *controllerServer {
 	return &controllerServer{
 		DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
-		clientSet:               clientSet,
 	}
 }
 
-func NewNodeServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *nodeServer {
+func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
 	return &nodeServer{
 		DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
-		clientSet:         clientSet,
 	}
 }
 
-func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.Clientset) {
+func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
 	glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
 
 	// Initialize default library driver
@@ -91,8 +88,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.C
 
 	// Create GRPC servers
 	rbd.ids = NewIdentityServer(rbd.driver)
-	rbd.ns = NewNodeServer(rbd.driver, clientSet)
-	rbd.cs = NewControllerServer(rbd.driver, clientSet)
+	rbd.ns = NewNodeServer(rbd.driver)
+	rbd.cs = NewControllerServer(rbd.driver)
 	s := csicommon.NewNonBlockingGRPCServer()
 	s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)
 	s.Wait()

pkg/rbd/rbd_util.go

@@ -21,9 +21,7 @@ import (
 	"fmt"
 	"github.com/golang/glog"
 	"io/ioutil"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/pkg/util/keymutex"
 	"os"
 	"os/exec"
@@ -50,17 +48,12 @@ type rbdVolumeOptions struct {
 	VolName       string   `json:"volName"`
 	Monitors      string   `json:"monitors"`
 	Pool          string   `json:"pool"`
-	AdminSecretName      string            `json:"adminSecret"`
-	AdminSecretNamespace string            `json:"adminSecretNamespace"`
 	AdminID       string   `json:"adminID"`
+	AdminSecret   string   `json:"adminSecret"`
 	UserID        string   `json:"userID"`
-	UserSecretName       string            `json:"userSecret"`
-	UserSecretNamespace  string            `json:"userSecretNamespace"`
+	UserSecret    string   `json:"userSecret"`
 	ImageFormat   string   `json:"imageFormat"`
 	ImageFeatures []string `json:"imageFeatures"`
-	ImageMapping         map[string]string `json:"imageMapping"`
-	adminSecret          string
-	userSecret           string
 }
 
 var attachdetachMutex = keymutex.NewKeyMutex()
@@ -76,11 +69,11 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
 	volSzGB := fmt.Sprintf("%dG", volSz)
 
 	if pOpts.ImageFormat == rbdImageFormat2 {
-		glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
+		glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
 	} else {
-		glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
+		glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
 	}
-	args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.adminSecret, "--image-format", pOpts.ImageFormat}
+	args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat}
 	if pOpts.ImageFormat == rbdImageFormat2 {
 		// if no image features is provided, it results in empty string
 		// which disable all RBD image format 2 features as we expected
@@ -106,10 +99,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
 	image := b.VolName
 	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
 	id := b.AdminID
-	secret := b.adminSecret
+	secret := b.AdminSecret
 	if id == "" {
 		id = b.UserID
-		secret = b.userSecret
+		secret = b.UserSecret
 	}
 
 	glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
@@ -152,10 +145,10 @@ func deleteRBDImage(b *rbdVolumeOptions) error {
 		return fmt.Errorf("rbd %s is still being used", image)
 	}
 	id := b.AdminID
-	secret := b.adminSecret
+	secret := b.AdminSecret
 	if id == "" {
 		id = b.UserID
-		secret = b.userSecret
+		secret = b.UserSecret
 	}
 
 	glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
@@ -173,25 +166,16 @@ func execCommand(command string, args []string) ([]byte, error) {
 	return cmd.CombinedOutput()
 }
 
-func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Clientset) (*rbdVolumeOptions, error) {
+func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) {
 	rbdVolume := &rbdVolumeOptions{}
 	var ok bool
-	var err error
-	rbdVolume.AdminID, ok = volOptions["adminId"]
+	rbdVolume.AdminID, ok = volOptions["adminID"]
 	if !ok {
-		return nil, fmt.Errorf("Missing required parameter adminId")
+		return nil, fmt.Errorf("Missing required parameter adminID")
 	}
-	rbdVolume.AdminSecretName, ok = volOptions["adminSecretName"]
+	rbdVolume.AdminSecret, ok = volOptions["adminSecret"]
 	if !ok {
-		return nil, fmt.Errorf("Missing required parameter adminSecretName")
-	}
-	rbdVolume.AdminSecretNamespace, ok = volOptions["adminSecretNamespace"]
-	if !ok {
-		rbdVolume.AdminSecretNamespace = "default"
-	}
-	rbdVolume.adminSecret, err = parseStorageClassSecret(rbdVolume.AdminSecretName, rbdVolume.AdminSecretNamespace, client)
-	if err != nil {
-		return nil, fmt.Errorf("Failed to retrieve Admin secret %v", err)
+		return nil, fmt.Errorf("Missing required parameter adminSecret")
 	}
 	rbdVolume.Pool, ok = volOptions["pool"]
 	if !ok {
@@ -201,23 +185,13 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client
 	if !ok {
 		return nil, fmt.Errorf("Missing required parameter monitors")
 	}
-	if err != nil {
-		return nil, err
-	}
-	rbdVolume.UserID, ok = volOptions["userId"]
+	rbdVolume.UserID, ok = volOptions["userID"]
 	if !ok {
-		return nil, fmt.Errorf("Missing required parameter userId")
+		return nil, fmt.Errorf("Missing required parameter userID")
 	}
-	rbdVolume.UserSecretName, ok = volOptions["userSecretName"]
-	if ok {
-		rbdVolume.UserSecretNamespace, ok = volOptions["userSecretNamespace"]
+	rbdVolume.UserSecret, ok = volOptions["userSecret"]
 	if !ok {
-			rbdVolume.UserSecretNamespace = "default"
-		}
-		rbdVolume.userSecret, err = parseStorageClassSecret(rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, client)
-		if err != nil {
-			glog.Errorf("failed to retrieve user's secret: %s/%s  (%v)", rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, err)
-		}
+		return nil, fmt.Errorf("Missing required parameter userSecret")
 	}
 	rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
 	if !ok {
@@ -227,25 +201,6 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client
 	return rbdVolume, nil
 }
 
-func parseStorageClassSecret(secretName string, namespace string, client *kubernetes.Clientset) (string, error) {
-	if client == nil {
-		return "", fmt.Errorf("Cannot get kube client")
-	}
-	secrets, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
-	if err != nil {
-		return "", err
-	}
-	secret := ""
-	for k, v := range secrets.Data {
-		if k == secretName {
-			return string(v), nil
-		}
-		secret = string(v)
-	}
-
-	return secret, nil
-}
-
 func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
 	var err error
 	var output []byte
@@ -283,13 +238,8 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
 		}
 
 		glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
-		// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
-		id := volOptions.AdminID
-		secret := volOptions.adminSecret
-		if id == "" {
-			id = volOptions.UserID
-			secret = volOptions.userSecret
-		}
+		id := volOptions.UserID
+		secret := volOptions.UserSecret
 
 		output, err = execCommand("rbd", []string{
 			"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
@@ -311,17 +261,12 @@ func detachRBDImage(volOptions *rbdVolumeOptions) error {
 	var output []byte
 
 	image := volOptions.VolName
-	glog.V(1).Infof("rbd: unmap device %s", volOptions.ImageMapping[image])
-	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
-	id := volOptions.AdminID
-	secret := volOptions.adminSecret
-	if id == "" {
-		id = volOptions.UserID
-		secret = volOptions.userSecret
-	}
+	glog.V(1).Infof("rbd: unmap device %s", image)
+	id := volOptions.UserID
+	secret := volOptions.UserSecret
 
 	output, err = execCommand("rbd", []string{
-		"unmap", volOptions.ImageMapping[image], "--id", id, "--key=" + secret})
+		"unmap", image, "--id", id, "--key=" + secret})
 	if err != nil {
 		glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output))
 		return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))

rbd/main.go

@@ -18,13 +18,11 @@ package main
 
 import (
 	"flag"
-	"github.com/golang/glog"
 	"os"
 	"path"
 
 	"github.com/ceph/ceph-csi/pkg/rbd"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
+	"github.com/golang/glog"
 )
 
 func init() {
@@ -40,17 +38,6 @@ var (
 func main() {
 	flag.Parse()
 
-	// creates the in-cluster config
-	config, err := rest.InClusterConfig()
-	if err != nil {
-		panic(err.Error())
-	}
-	// creates the clientset
-	clientSet, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		panic(err.Error())
-	}
-
 	if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {
 		glog.Errorf("failed to create persistent storage for controller %v", err)
 		os.Exit(1)
@@ -60,13 +47,13 @@ func main() {
 		os.Exit(1)
 	}
 
-	handle(clientSet)
+	handle()
 	os.Exit(0)
 }
 
-func handle(clientSet *kubernetes.Clientset) {
+func handle() {
 	driver := rbd.GetRBDDriver()
-	driver.Run(*driverName, *nodeID, *endpoint, clientSet)
+	driver.Run(*driverName, *nodeID, *endpoint)
 }
 
 func createPersistentStorage(persistentStoragePath string) error {