From b0e68a52e0e533523cd9bf7e2f06aa894ffeafdc Mon Sep 17 00:00:00 2001 From: Seungcheol Ko Date: Thu, 9 Aug 2018 22:07:00 +0900 Subject: [PATCH] Refactoring using users --- examples/rbd/secret.yaml | 2 + examples/rbd/storageclass.yaml | 4 ++ pkg/rbd/controllerserver.go | 18 +++---- pkg/rbd/nodeserver.go | 2 +- pkg/rbd/rbd.go | 8 ++- pkg/rbd/rbd_util.go | 98 ++++++++++++++++++++-------------- 6 files changed, 78 insertions(+), 54 deletions(-) diff --git a/examples/rbd/secret.yaml b/examples/rbd/secret.yaml index 594fd8ab4..f15cbcbfd 100644 --- a/examples/rbd/secret.yaml +++ b/examples/rbd/secret.yaml @@ -6,3 +6,5 @@ metadata: data: # Key value corresponds to a user name defined in ceph cluster admin: BASE64-ENCODED-PASSWORD + # Key value corresponds to a user name defined in ceph cluster + kubernetes: BASE64-ENCODED-PASSWORD diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 476c2bef3..e1a8a06ba 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -21,4 +21,8 @@ parameters: csiProvisionerSecretNamespace: default csiNodePublishSecretName: csi-rbd-secret csiNodePublishSecretNamespace: default + + # Ceph users for operating RBD + adminid: admin + userid: kubernetes reclaimPolicy: Delete diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 9dbe1f55a..ba0e307eb 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -98,7 +98,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024) // Check if there is already RBD image with requested name - found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets()) + found, _, _ := rbdStatus(rbdVol, rbdVol.UserId, req.GetControllerCreateSecrets()) if !found { // if VolumeContentSource is not nil, this request is for snapshot if req.VolumeContentSource != nil { @@ -117,13 +117,13 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - err = restoreSnapshot(rbdVol, rbdSnap, req.GetControllerCreateSecrets()) + err = restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminId, req.GetControllerCreateSecrets()) if err != nil { return nil, err } glog.V(4).Infof("create volume %s from snapshot %s", volName, rbdSnap.SnapName) } else { - if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { + if err := createRBDImage(rbdVol, volSizeGB, rbdVol.AdminId, req.GetControllerCreateSecrets()); err != nil { if err != nil { glog.Warningf("failed to create volume: %v", err) return nil, err @@ -161,7 +161,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol volName := rbdVol.VolName // Deleting rbd image glog.V(4).Infof("deleting volume %s", volName) - if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil { + if err := deleteRBDImage(rbdVol, rbdVol.AdminId, req.GetControllerDeleteSecrets()); err != nil { glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err) return nil, err } @@ -243,7 +243,7 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS rbdSnap.SourceVolumeID = req.GetSourceVolumeId() rbdSnap.SizeBytes = rbdVolume.VolSize - err = createSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = createSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) // if we already have the snapshot, return the snapshot if err != nil { if exitErr, ok := err.(*exec.ExitError); ok { @@ -264,10 +264,10 @@ func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS } } else { glog.V(4).Infof("create snapshot %s", snapName) - err = protectSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = protectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) if err != nil { - err = deleteSnapshot(rbdSnap, req.GetCreateSnapshotSecrets()) + err = deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetCreateSnapshotSecrets()) if err != nil { return nil, fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) } @@ -313,14 +313,14 @@ func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS } // Unprotect snapshot - err := unprotectSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()) + err := unprotectSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets()) if err != nil { return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) } // Deleting snapshot glog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) - if err := deleteSnapshot(rbdSnap, req.GetDeleteSnapshotSecrets()); err != nil { + if err := deleteSnapshot(rbdSnap, rbdSnap.AdminId, req.GetDeleteSnapshotSecrets()); err != nil { return nil, status.Error(codes.FailedPrecondition, fmt.Sprintf("failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err)) } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 7091ca99c..ca309b18e 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } volOptions.VolName = volName // Mapping RBD image - devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets()) + devicePath, err := attachRBDImage(volOptions, volOptions.UserId, req.GetNodePublishSecrets()) if err != nil { return nil, err } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index d5c6d000e..3f0a04f70 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -31,11 +31,9 @@ import ( // PluginFolder defines the location of rbdplugin const ( - PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" - // RBDUserID used as a key in credentials map to extract the key which is - // passed be the provisioner, the value od RBDUserID must match to the key used - // in Secret object. - RBDUserID = "admin" + PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" + rbdDefaultAdminId = "admin" + rbdDefaultUserId = rbdDefaultAdminId ) type rbd struct { diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 66c4d1da4..7b0c386f6 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -54,6 +54,8 @@ type rbdVolume struct { ImageFormat string `json:"imageFormat"` ImageFeatures string `json:"imageFeatures"` VolSize int64 `json:"volSize"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` } type rbdSnapshot struct { @@ -65,6 +67,8 @@ type rbdSnapshot struct { Pool string `json:"pool"` CreatedAt int64 `json:"createdAt"` SizeBytes int64 `json:"sizeBytes"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` } var ( @@ -81,7 +85,7 @@ func getRBDKey(id string, credentials map[string]string) (string, error) { } // CreateImage creates a new ceph image with provision and volume options. -func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error { +func createRBDImage(pOpts *rbdVolume, volSz int, adminId string, credentials map[string]string) error { var output []byte var err error @@ -90,16 +94,16 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) image := pOpts.VolName volSzGB := fmt.Sprintf("%dG", volSz) - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } if pOpts.ImageFormat == rbdImageFormat2 { - glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key) + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, adminId, key) } else { - glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key) + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, adminId, key) } - args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} + args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", adminId, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { args = append(args, "--image-feature", pOpts.ImageFeatures) } @@ -114,20 +118,21 @@ func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) // rbdStatus checks if there is watcher on the image. // It returns true if there is a watcher onthe image, otherwise returns false. -func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) { +func rbdStatus(pOpts *rbdVolume, userId string, credentials map[string]string) (bool, string, error) { var err error var output string var cmd []byte image := pOpts.VolName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - key, err := getRBDKey(RBDUserID, credentials) + + key, err := getRBDKey(userId, credentials) if err != nil { return false, "", err } - glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key} + glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, userId, key) + args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userId, "--key=" + key} cmd, err = execCommand("rbd", args) output = string(cmd) @@ -154,10 +159,10 @@ func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, e } // DeleteImage deletes a ceph image with provision and volume options. -func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error { +func deleteRBDImage(pOpts *rbdVolume, adminId string, credentials map[string]string) error { var output []byte image := pOpts.VolName - found, _, err := rbdStatus(pOpts, credentials) + found, _, err := rbdStatus(pOpts, adminId, credentials) if err != nil { return err } @@ -165,13 +170,13 @@ func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error { glog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key} + glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminId, "-m", pOpts.Monitors, "--key=" + key} output, err = execCommand("rbd", args) if err == nil { return nil @@ -215,7 +220,14 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { } } - + rbdVol.AdminId, ok = volOptions["adminid"] + if !ok { + rbdVol.AdminId = rbdDefaultAdminId + } + rbdVol.UserId, ok = volOptions["userid"] + if !ok { + rbdVol.UserId = rbdDefaultUserId + } return rbdVol, nil } @@ -230,11 +242,19 @@ func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) if !ok { return nil, fmt.Errorf("Missing required parameter monitors") } + rbdSnap.AdminId, ok = snapOptions["adminid"] + if !ok { + rbdSnap.AdminId = rbdDefaultAdminId + } + rbdSnap.UserId, ok = snapOptions["userid"] + if !ok { + rbdSnap.UserId = rbdDefaultUserId + } return rbdSnap, nil } -func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) { +func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string]string) (string, error) { var err error var output []byte @@ -255,7 +275,7 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin Steps: rbdImageWatcherSteps, } err := wait.ExponentialBackoff(backoff, func() (bool, error) { - used, rbdOutput, err := rbdStatus(volOptions, credentials) + used, rbdOutput, err := rbdStatus(volOptions, userId, credentials) if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } @@ -271,13 +291,13 @@ func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (strin } glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(userId, credentials) if err != nil { return "", err } output, err = execCommand("rbd", []string{ - "map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key}) + "map", image, "--pool", volOptions.Pool, "--id", userId, "-m", volOptions.Monitors, "--key=" + key}) if err != nil { glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) @@ -471,7 +491,7 @@ func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) { return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) } -func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -479,12 +499,12 @@ func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -495,7 +515,7 @@ func protectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -503,12 +523,12 @@ func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -519,7 +539,7 @@ func createSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -527,12 +547,12 @@ func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -543,7 +563,7 @@ func unprotectSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error return nil } -func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { +func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -551,12 +571,12 @@ func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) - args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -567,7 +587,7 @@ func deleteSnapshot(pOpts *rbdSnapshot, credentials map[string]string) error { return nil } -func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials map[string]string) error { +func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte var err error @@ -575,12 +595,12 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, credentials ma image := pVolOpts.VolName snapID := pSnapOpts.SnapID - key, err := getRBDKey(RBDUserID, credentials) + key, err := getRBDKey(adminId, credentials) if err != nil { return err } - glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, RBDUserID, key) - args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", RBDUserID, "-m", mon, "--key=" + key} + glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, adminId, key) + args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args)