diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index ca9468707..ffab3f4c8 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -33,7 +33,8 @@ Option | Default value | Description Parameter | Required | Description --------- | -------- | ----------- -`monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`) +`monitors` | one of `monitors` and `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`) +`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. `pool` | yes | Ceph pool into which the RBD image shall be created `imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format) `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) diff --git a/examples/rbd/secret.yaml b/examples/rbd/secret.yaml index f15cbcbfd..60f1d11d6 100644 --- a/examples/rbd/secret.yaml +++ b/examples/rbd/secret.yaml @@ -8,3 +8,6 @@ data: admin: BASE64-ENCODED-PASSWORD # Key value corresponds to a user name defined in ceph cluster kubernetes: BASE64-ENCODED-PASSWORD + # if monValueFromSecret is set to "monitors", uncomment the + # following and set the mon there + #monitors: BASE64-ENCODED-Comma-Delimited-Mons diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 536b07095..98960a53d 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -8,6 +8,12 @@ parameters: # if using FQDN, make sure csi plugin's dns policy is appropriate. monitors: mon1:port,mon2:port,... + # if "monitors" parameter is not set, driver to get monitors from same + # secret as admin/user credentials. "monValueFromSecret" provides the + # key in the secret whose value is the mons + #monValueFromSecret: "monitors" + + # Ceph pool into which the RBD image shall be created pool: rbd diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 6385db81a..c9611c09f 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -266,13 +266,18 @@ func attachRBDImage(volOptions *rbdVolume, userId string, credentials map[string return "", err } - glog.V(3).Infof("rbd: map mon %s", volOptions.Monitors) + mon, err := getMon(volOptions, credentials) + if err != nil { + return "", err + } + + glog.V(5).Infof("rbd: map mon %s", mon) key, err := getRBDKey(userId, credentials) if err != nil { return "", err } output, err = execCommand(cmdName, []string{ - "map", imagePath, "--id", userId, "-m", volOptions.Monitors, "--key=" + key}) + "map", imagePath, "--id", userId, "-m", mon, "--key=" + key}) if err != nil { glog.Warningf("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index eaea663d7..8012c4325 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -46,29 +46,31 @@ const ( ) type rbdVolume struct { - VolName string `json:"volName"` - VolID string `json:"volID"` - Monitors string `json:"monitors"` - Pool string `json:"pool"` - ImageFormat string `json:"imageFormat"` - ImageFeatures string `json:"imageFeatures"` - VolSize int64 `json:"volSize"` - AdminId string `json:"adminId"` - UserId string `json:"userId"` - Mounter string `json:"mounter"` + VolName string `json:"volName"` + VolID string `json:"volID"` + Monitors string `json:"monitors"` + MonValueFromSecret string `json:"monValueFromSecret"` + Pool string `json:"pool"` + ImageFormat string `json:"imageFormat"` + ImageFeatures string `json:"imageFeatures"` + VolSize int64 `json:"volSize"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` + Mounter string `json:"mounter"` } type rbdSnapshot struct { - SourceVolumeID string `json:"sourceVolumeID"` - VolName string `json:"volName"` - SnapName string `json:"snapName"` - SnapID string `json:"sanpID"` - Monitors string `json:"monitors"` - Pool string `json:"pool"` - CreatedAt int64 `json:"createdAt"` - SizeBytes int64 `json:"sizeBytes"` - AdminId string `json:"adminId"` - UserId string `json:"userId"` + SourceVolumeID string `json:"sourceVolumeID"` + VolName string `json:"volName"` + SnapName string `json:"snapName"` + SnapID string `json:"sanpID"` + Monitors string `json:"monitors"` + MonValueFromSecret string `json:"monValueFromSecret"` + Pool string `json:"pool"` + CreatedAt int64 `json:"createdAt"` + SizeBytes int64 `json:"sizeBytes"` + AdminId string `json:"adminId"` + UserId string `json:"userId"` } var ( @@ -84,13 +86,32 @@ func getRBDKey(id string, credentials map[string]string) (string, error) { return "", fmt.Errorf("RBD key for ID: %s not found", id) } +func getMon(pOpts *rbdVolume, credentials map[string]string) (string, error) { + mon := pOpts.Monitors + if len(mon) == 0 { + // if mons are set in secret, retrieve them + if len(pOpts.MonValueFromSecret) == 0 { + // yet another sanity check + return "", fmt.Errorf("either monitors or monValueFromSecret must be set") + } + if val, ok := credentials[pOpts.MonValueFromSecret]; !ok { + return "", fmt.Errorf("mon data %s is not set in secret", pOpts.MonValueFromSecret) + } else { + mon = val + } + } + return mon, nil +} + // CreateImage creates a new ceph image with provision and volume options. func createRBDImage(pOpts *rbdVolume, volSz int, adminId string, credentials map[string]string) error { var output []byte - var err error - // rbd create - mon := pOpts.Monitors + mon, err := getMon(pOpts, credentials) + if err != nil { + return err + } + image := pOpts.VolName volSzGB := fmt.Sprintf("%dG", volSz) @@ -119,7 +140,6 @@ func createRBDImage(pOpts *rbdVolume, volSz int, adminId string, credentials map // rbdStatus checks if there is watcher on the image. // It returns true if there is a watcher onthe image, otherwise returns false. func rbdStatus(pOpts *rbdVolume, userId string, credentials map[string]string) (bool, string, error) { - var err error var output string var cmd []byte @@ -131,8 +151,13 @@ func rbdStatus(pOpts *rbdVolume, userId string, credentials map[string]string) ( return false, "", err } - glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, userId, key) - args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userId, "--key=" + key} + mon, err := getMon(pOpts, credentials) + if err != nil { + return false, "", err + } + + glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, userId, key) + args := []string{"status", image, "--pool", pOpts.Pool, "-m", mon, "--id", userId, "--key=" + key} cmd, err = execCommand("rbd", args) output = string(cmd) @@ -174,9 +199,13 @@ func deleteRBDImage(pOpts *rbdVolume, adminId string, credentials map[string]str if err != nil { return err } + mon, err := getMon(pOpts, credentials) + if err != nil { + return err + } - glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) - args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminId, "-m", pOpts.Monitors, "--key=" + key} + glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, adminId, key) + args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) if err == nil { return nil @@ -199,7 +228,10 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { } rbdVol.Monitors, ok = volOptions["monitors"] if !ok { - return nil, fmt.Errorf("Missing required parameter monitors") + // if mons are not set in options, check if they are set in secret + if rbdVol.MonValueFromSecret, ok = volOptions["monValueFromSecret"]; !ok { + return nil, fmt.Errorf("Either monitors or monValueFromSecret must be set") + } } rbdVol.ImageFormat, ok = volOptions["imageFormat"] if !ok { @@ -244,7 +276,10 @@ func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) } rbdSnap.Monitors, ok = snapOptions["monitors"] if !ok { - return nil, fmt.Errorf("Missing required parameter monitors") + // if mons are not set in options, check if they are set in secret + if rbdSnap.MonValueFromSecret, ok = snapOptions["monValueFromSecret"]; !ok { + return nil, fmt.Errorf("Either monitors or monValueFromSecret must be set") + } } rbdSnap.AdminId, ok = snapOptions["adminid"] if !ok { @@ -382,11 +417,26 @@ func getRBDSnapshotByName(snapName string) (*rbdSnapshot, error) { return nil, fmt.Errorf("snapshot name %s does not exit in the snapshots list", snapName) } +func getSnapMon(pOpts *rbdSnapshot, credentials map[string]string) (string, error) { + mon := pOpts.Monitors + if len(mon) == 0 { + // if mons are set in secret, retrieve them + if len(pOpts.MonValueFromSecret) == 0 { + // yet another sanity check + return "", fmt.Errorf("either monitors or monValueFromSecret must be set") + } + if val, ok := credentials[pOpts.MonValueFromSecret]; !ok { + return "", fmt.Errorf("mon data %s is not set in secret", pOpts.MonValueFromSecret) + } else { + mon = val + } + } + return mon, nil +} + func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte - var err error - mon := pOpts.Monitors image := pOpts.VolName snapID := pOpts.SnapID @@ -394,7 +444,12 @@ func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string] if err != nil { return err } - glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + mon, err := getSnapMon(pOpts, credentials) + if err != nil { + return err + } + + glog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, adminId, key) args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -408,9 +463,12 @@ func protectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string] func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte - var err error - mon := pOpts.Monitors + mon, err := getSnapMon(pOpts, credentials) + if err != nil { + return err + } + image := pOpts.VolName snapID := pOpts.SnapID @@ -418,7 +476,7 @@ func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]s if err != nil { return err } - glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + glog.V(4).Infof("rbd: snap create %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, adminId, key) args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -432,9 +490,12 @@ func createSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]s func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte - var err error - mon := pOpts.Monitors + mon, err := getSnapMon(pOpts, credentials) + if err != nil { + return err + } + image := pOpts.VolName snapID := pOpts.SnapID @@ -442,7 +503,7 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[strin if err != nil { return err } - glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + glog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, adminId, key) args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -456,9 +517,12 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[strin func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte - var err error - mon := pOpts.Monitors + mon, err := getSnapMon(pOpts, credentials) + if err != nil { + return err + } + image := pOpts.VolName snapID := pOpts.SnapID @@ -466,7 +530,7 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]s if err != nil { return err } - glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, adminId, key) + glog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s id %s key %s", image, mon, pOpts.Pool, adminId, key) args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapID, image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args) @@ -480,9 +544,12 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminId string, credentials map[string]s func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminId string, credentials map[string]string) error { var output []byte - var err error - mon := pVolOpts.Monitors + mon, err := getMon(pVolOpts, credentials) + if err != nil { + return err + } + image := pVolOpts.VolName snapID := pSnapOpts.SnapID @@ -490,7 +557,7 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminId string if err != nil { return err } - glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, pVolOpts.Monitors, pVolOpts.Pool, adminId, key) + glog.V(4).Infof("rbd: clone %s using mon %s, pool %s id %s key %s", image, mon, pVolOpts.Pool, adminId, key) args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.VolName + "@" + snapID, pVolOpts.Pool + "/" + image, "--id", adminId, "-m", mon, "--key=" + key} output, err = execCommand("rbd", args)