mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-22 14:20:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Modify RBD plugin to use a single ID and move the id and key into the secret

Browse Source 
RBD plugin needs only a single ID to manage images and operations against a
pool, mentioned in the storage class. The current scheme of 2 IDs is hence not
needed and removed in this commit.

Further, unlike CephFS plugin, the RBD plugin splits the user id and the key
into the storage class and the secret respectively. Also the parameter name
for the key in the secret is noted in the storageclass making it a variant and
hampers usability/comprehension. This is also fixed by moving the id and the key
to the secret and not retaining the same in the storage class, like CephFS.

Fixes #270

Testing done:
- Basic PVC creation and mounting

Signed-off-by: ShyamsundarR <srangana@redhat.com>
This commit is contained in:
ShyamsundarR 2019-06-01 17:26:42 -04:00 committed by mergify[bot]
parent 22ff5c0911
commit c5762b6b5c
25 changed files with 284 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
docs/deploy-rbd.md
View File

@ -61,9 +61,8 @@ provisioning.
**Required secrets:**
Admin credentials are required for provisioning new RBD images `ADMIN_NAME`:
`ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the
client with admin privileges, and the value is its password
User credentials, with required access to the pool being used in the storage class,
is required for provisioning new RBD images.
## Deployment with Kubernetes

13
e2e/utils.go
View File

@ -193,7 +193,6 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework) {
func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework) {
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml")
sc := getStorageClass(scPath)
delete(sc.Parameters, "userid")
sc.Parameters["pool"] = "replicapool"
opt := metav1.ListOptions{
LabelSelector: "app=rook-ceph-tools",
@ -283,10 +282,10 @@ func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) {
LabelSelector: "app=rook-ceph-tools",
}
adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt)
sc.Data["adminID"] = []byte("admin")
sc.Data["adminKey"] = []byte(adminKey)
delete(sc.Data, "userID")
delete(sc.Data, "userKey")
sc.StringData["adminID"] = "admin"
sc.StringData["adminKey"] = adminKey
delete(sc.StringData, "userID")
delete(sc.StringData, "userKey")
_, err := c.CoreV1().Secrets("default").Create(&sc)
Expect(err).Should(BeNil())
}
@ -298,8 +297,8 @@ func createRBDSecret(c kubernetes.Interface, f *framework.Framework) {
LabelSelector: "app=rook-ceph-tools",
}
adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt)
sc.Data["admin"] = []byte(adminKey)
delete(sc.Data, "kubernetes")
sc.StringData["userID"] = "admin"
sc.StringData["userKey"] = adminKey
_, err := c.CoreV1().Secrets("default").Create(&sc)
Expect(err).Should(BeNil())
}

10
examples/cephfs/secret.yaml
View File

@ -4,11 +4,11 @@ kind: Secret
metadata:
name: csi-cephfs-secret
namespace: default
data:
stringData:
# Required for statically provisioned volumes
userID: BASE64-ENCODED-VALUE
userKey: BASE64-ENCODED-VALUE
userID: <plaintext ID>
userKey: <Ceph auth key corresponding to ID above>
# Required for dynamically provisioned volumes
adminID: BASE64-ENCODED-VALUE
adminKey: BASE64-ENCODED-VALUE
adminID: <plaintext ID>
adminKey: <Ceph auth key corresponding to ID above>

11
examples/rbd/secret.yaml
View File

@ -4,8 +4,9 @@ kind: Secret
metadata:
name: csi-rbd-secret
namespace: default
data:
# Key value corresponds to a user name defined in ceph cluster
admin: BASE64-ENCODED-PASSWORD
# Key value corresponds to a user name defined in ceph cluster
kubernetes: BASE64-ENCODED-PASSWORD
stringData:
# Key values correspond to a user name and its key, as defined in the
# ceph cluster. User ID should have required access to the 'pool'
# specified in the storage class
userID: <plaintext ID>
userKey: <Ceph auth key corresponding to ID above>

7
examples/rbd/storageclass.yaml
View File

@ -24,16 +24,13 @@ parameters:
# CSI RBD currently supports only `layering` feature.
imageFeatures: layering
# The secrets have to contain Ceph admin credentials.
# The secrets have to contain Ceph credentials with required access
# to the 'pool'.
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-publish-secret-namespace: default
# Ceph users for operating RBD
adminid: admin
userid: kubernetes
# uncomment the following to use rbd-nbd as mounter on supported nodes
# mounter: rbd-nbd
reclaimPolicy: Delete

18
pkg/cephfs/cephfs_util.go
View File

@ -33,15 +33,15 @@ type CephFilesystemDetails struct {
MDSMap MDSMap `json:"mdsmap"`
}
func getFscID(monitors, id, key, fsName string) (int64, error) {
func getFscID(monitors string, cr *util.Credentials, fsName string) (int64, error) {
// ceph fs get myfs --format=json
// {"mdsmap":{...},"id":2}
var fsDetails CephFilesystemDetails
err := execCommandJSON(&fsDetails,
"ceph",
"-m", monitors,
"--id", id,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", util.CephConfigPath,
"fs", "get", fsName, "--format=json",
)
@ -61,15 +61,15 @@ type CephFilesystem struct {
DataPoolIDs []int `json:"data_pool_ids"`
}
func getMetadataPool(monitors, id, key, fsName string) (string, error) {
func getMetadataPool(monitors string, cr *util.Credentials, fsName string) (string, error) {
// ./tbox ceph fs ls --format=json
// [{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,...},...]
var filesystems []CephFilesystem
err := execCommandJSON(&filesystems,
"ceph",
"-m", monitors,
"--id", id,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", util.CephConfigPath,
"fs", "ls", "--format=json",
)
@ -91,15 +91,15 @@ type CephFilesystemDump struct {
Filesystems []CephFilesystemDetails `json:"filesystems"`
}
func getFsName(monitors, id, key string, fscID int64) (string, error) {
func getFsName(monitors string, cr *util.Credentials, fscID int64) (string, error) {
// ./tbox ceph fs dump --format=json
// JSON: {...,"filesystems":[{"mdsmap":{},"id":<n>},...],...}
var fsDump CephFilesystemDump
err := execCommandJSON(&fsDump,
"ceph",
"-m", monitors,
"--id", id,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", util.CephConfigPath,
"fs", "dump", "--format=json",
)

24
pkg/cephfs/cephuser.go
View File

@ -39,10 +39,10 @@ type cephEntity struct {
Caps cephEntityCaps `json:"caps"`
}
func (ent *cephEntity) toCredentials() *credentials {
return &credentials{
id: ent.Entity[len(cephEntityClientPrefix):],
key: ent.Key,
func (ent *cephEntity) toCredentials() *util.Credentials {
return &util.Credentials{
ID: ent.Entity[len(cephEntityClientPrefix):],
Key: ent.Key,
}
}
@ -63,30 +63,30 @@ func getSingleCephEntity(args ...string) (*cephEntity, error) {
return &ents[0], nil
}
func genUserIDs(adminCr *credentials, volID volumeID) (adminID, userID string) {
return cephEntityClientPrefix + adminCr.id, cephEntityClientPrefix + getCephUserName(volID)
func genUserIDs(adminCr *util.Credentials, volID volumeID) (adminID, userID string) {
return cephEntityClientPrefix + adminCr.ID, cephEntityClientPrefix + getCephUserName(volID)
}
func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
func getCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) (*cephEntity, error) {
adminID, userID := genUserIDs(adminCr, volID)
return getSingleCephEntity(
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"--key="+adminCr.Key,
"-c", util.CephConfigPath,
"-f", "json",
"auth", "get", userID,
)
}
func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
func createCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) (*cephEntity, error) {
adminID, userID := genUserIDs(adminCr, volID)
return getSingleCephEntity(
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"--key="+adminCr.Key,
"-c", util.CephConfigPath,
"-f", "json",
"auth", "get-or-create", userID,
@ -97,14 +97,14 @@ func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volum
)
}
func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) error {
func deleteCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) error {
adminID, userID := genUserIDs(adminCr, volID)
// TODO: Need to return success if userID is not found
return execCommandErr("ceph",
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"--key="+adminCr.Key,
"-c", util.CephConfigPath,
"auth", "rm", userID,
)

8
pkg/cephfs/controllerserver.go
View File

@ -48,7 +48,7 @@ var (
// createBackingVolume creates the backing subvolume and user/key for the given volOptions and vID,
// and on any error cleans up any created entities
func (cs *ControllerServer) createBackingVolume(volOptions *volumeOptions, vID *volumeIdentifier, secret map[string]string) error {
cr, err := getAdminCredentials(secret)
cr, err := util.GetAdminCredentials(secret)
if err != nil {
return status.Error(codes.InvalidArgument, err.Error())
}
@ -168,14 +168,14 @@ func (cs *ControllerServer) deleteVolumeDeprecated(req *csi.DeleteVolumeRequest)
// mons may have changed since create volume,
// retrieve the latest mons and override old mons
if mon, secretsErr := getMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 {
if mon, secretsErr := util.GetMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 {
klog.Infof("overriding monitors [%q] with [%q] for volume %s", ce.VolOptions.Monitors, mon, volID)
ce.VolOptions.Monitors = mon
}
// Deleting a volume requires admin credentials
cr, err := getAdminCredentials(secrets)
cr, err := util.GetAdminCredentials(secrets)
if err != nil {
klog.Errorf("failed to retrieve admin credentials: %v", err)
return nil, status.Error(codes.InvalidArgument, err.Error())
@ -232,7 +232,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
}
// Deleting a volume requires admin credentials
cr, err := getAdminCredentials(secrets)
cr, err := util.GetAdminCredentials(secrets)
if err != nil {
klog.Errorf("failed to retrieve admin credentials: %v", err)
return nil, status.Error(codes.InvalidArgument, err.Error())

12
pkg/cephfs/fsjournal.go
View File

@ -49,12 +49,12 @@ func checkVolExists(volOptions *volumeOptions, secret map[string]string) (*volum
vid volumeIdentifier
)
cr, err := getAdminCredentials(secret)
cr, err := util.GetAdminCredentials(secret)
if err != nil {
return nil, err
}
imageUUID, err := volJournal.CheckReservation(volOptions.Monitors, cr.id, cr.key,
imageUUID, err := volJournal.CheckReservation(volOptions.Monitors, cr,
volOptions.MetadataPool, volOptions.RequestName, "")
if err != nil {
return nil, err
@ -86,12 +86,12 @@ func checkVolExists(volOptions *volumeOptions, secret map[string]string) (*volum
// undoVolReservation is a helper routine to undo a name reservation for a CSI VolumeName
func undoVolReservation(volOptions *volumeOptions, vid volumeIdentifier, secret map[string]string) error {
cr, err := getAdminCredentials(secret)
cr, err := util.GetAdminCredentials(secret)
if err != nil {
return err
}
err = volJournal.UndoReservation(volOptions.Monitors, cr.id, cr.key, volOptions.MetadataPool,
err = volJournal.UndoReservation(volOptions.Monitors, cr, volOptions.MetadataPool,
vid.FsSubvolName, volOptions.RequestName)
return err
@ -105,12 +105,12 @@ func reserveVol(volOptions *volumeOptions, secret map[string]string) (*volumeIde
vid volumeIdentifier
)
cr, err := getAdminCredentials(secret)
cr, err := util.GetAdminCredentials(secret)
if err != nil {
return nil, err
}
imageUUID, err := volJournal.ReserveName(volOptions.Monitors, cr.id, cr.key,
imageUUID, err := volJournal.ReserveName(volOptions.Monitors, cr,
volOptions.MetadataPool, volOptions.RequestName, "")
if err != nil {
return nil, err

6
pkg/cephfs/mountcache.go
View File

@ -90,13 +90,13 @@ func mountOneCacheEntry(volOptions *volumeOptions, vid *volumeIdentifier, me *vo
var (
err error
cr *credentials
cr *util.Credentials
)
volID := vid.VolumeID
if volOptions.ProvisionVolume {
volOptions.RootPath = getVolumeRootPathCeph(volumeID(vid.FsSubvolName))
cr, err = getAdminCredentials(decodeCredentials(me.Secrets))
cr, err = util.GetAdminCredentials(decodeCredentials(me.Secrets))
if err != nil {
return err
}
@ -107,7 +107,7 @@ func mountOneCacheEntry(volOptions *volumeOptions, vid *volumeIdentifier, me *vo
}
cr = entity.toCredentials()
} else {
cr, err = getUserCredentials(decodeCredentials(me.Secrets))
cr, err = util.GetUserCredentials(decodeCredentials(me.Secrets))
if err != nil {
return err
}

9
pkg/cephfs/nodeserver.go
View File

@ -22,6 +22,7 @@ import (
"os"
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
@ -40,9 +41,9 @@ var (
mtxNodeVolumeID = keymutex.NewHashed(0)
)
func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) {
func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*util.Credentials, error) {
var (
cr *credentials
cr *util.Credentials
secrets = req.GetSecrets()
)
@ -51,7 +52,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi
// First, get admin credentials - those are needed for retrieving the user credentials
adminCr, err := getAdminCredentials(secrets)
adminCr, err := util.GetAdminCredentials(secrets)
if err != nil {
return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
}
@ -67,7 +68,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi
} else {
// The volume is pre-made, credentials are in node stage secrets
userCr, err := getUserCredentials(req.GetSecrets())
userCr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
}

8
pkg/cephfs/volume.go
View File

@ -21,6 +21,8 @@ import (
"os"
"path"
"github.com/ceph/ceph-csi/pkg/util"
"k8s.io/klog"
)
@ -50,7 +52,7 @@ func setVolumeAttribute(root, attrName, attrValue string) error {
return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root)
}
func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeID, bytesQuota int64) error {
func createVolume(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID, bytesQuota int64) error {
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
return err
}
@ -91,7 +93,7 @@ func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeI
return nil
}
func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions) error {
func purgeVolume(volID volumeID, adminCr *util.Credentials, volOptions *volumeOptions) error {
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
return err
}
@ -120,7 +122,7 @@ func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions
return nil
}
func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *credentials) error {
func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *util.Credentials) error {
cephRoot := getCephRootPathLocal(volID)
// Root path is not set for dynamically provisioned volumes

14
pkg/cephfs/volumemounter.go
View File

@ -70,7 +70,7 @@ func loadAvailableMounters() error {
}
type volumeMounter interface {
mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error
mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error
name() string
}
@ -114,12 +114,12 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) {
type fuseMounter struct{}
func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
func mountFuse(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
args := []string{
mountPoint,
"-m", volOptions.Monitors,
"-c", util.CephConfigPath,
"-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key,
"-n", cephEntityClientPrefix + cr.ID, "--key=" + cr.Key,
"-r", volOptions.RootPath,
"-o", "nonempty",
}
@ -154,7 +154,7 @@ func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) er
return nil
}
func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
func (m *fuseMounter) mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
if err := createMountPoint(mountPoint); err != nil {
return err
}
@ -166,7 +166,7 @@ func (m *fuseMounter) name() string { return "Ceph FUSE driver" }
type kernelMounter struct{}
func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
func mountKernel(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
if err := execCommandErr("modprobe", "ceph"); err != nil {
return err
}
@ -176,7 +176,7 @@ func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions)
fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
mountPoint,
}
optionsStr := fmt.Sprintf("name=%s,secret=%s", cr.id, cr.key)
optionsStr := fmt.Sprintf("name=%s,secret=%s", cr.ID, cr.Key)
if volOptions.FsName != "" {
optionsStr += fmt.Sprintf(",mds_namespace=%s", volOptions.FsName)
}
@ -185,7 +185,7 @@ func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions)
return execCommandErr("mount", args[:]...)
}
func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
func (m *kernelMounter) mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error {
if err := createMountPoint(mountPoint); err != nil {
return err
}

17
pkg/cephfs/volumeoptions.go
View File

@ -149,17 +149,17 @@ func newVolumeOptions(requestName string, size int64, volOptions, secret map[str
opts.RequestName = requestName
opts.Size = size
cr, err := getAdminCredentials(secret)
cr, err := util.GetAdminCredentials(secret)
if err != nil {
return nil, err
}
opts.FscID, err = getFscID(opts.Monitors, cr.id, cr.key, opts.FsName)
opts.FscID, err = getFscID(opts.Monitors, cr, opts.FsName)
if err != nil {
return nil, err
}
opts.MetadataPool, err = getMetadataPool(opts.Monitors, cr.id, cr.key, opts.FsName)
opts.MetadataPool, err = getMetadataPool(opts.Monitors, cr, opts.FsName)
if err != nil {
return nil, err
}
@ -194,23 +194,22 @@ func newVolumeOptionsFromVolID(volID string, volOpt, secrets map[string]string)
return nil, nil, errors.Wrapf(err, "failed to fetch monitor list using clusterID (%s)", vi.ClusterID)
}
cr, err := getAdminCredentials(secrets)
cr, err := util.GetAdminCredentials(secrets)
if err != nil {
return nil, nil, err
}
volOptions.FsName, err = getFsName(volOptions.Monitors, cr.id, cr.key, volOptions.FscID)
volOptions.FsName, err = getFsName(volOptions.Monitors, cr, volOptions.FscID)
if err != nil {
return nil, nil, err
}
volOptions.MetadataPool, err = getMetadataPool(volOptions.Monitors, cr.id, cr.key,
volOptions.FsName)
volOptions.MetadataPool, err = getMetadataPool(volOptions.Monitors, cr, volOptions.FsName)
if err != nil {
return nil, nil, err
}
volOptions.RequestName, _, err = volJournal.GetObjectUUIDData(volOptions.Monitors, cr.id, cr.key,
volOptions.RequestName, _, err = volJournal.GetObjectUUIDData(volOptions.Monitors, cr,
volOptions.MetadataPool, vi.ObjectUUID, false)
if err != nil {
return nil, nil, err
@ -250,7 +249,7 @@ func newVolumeOptionsFromVersion1Context(volID string, options, secrets map[stri
// check if there are mon values in secret and if so override option retrieved monitors from
// monitors in the secret
mon, err := getMonValFromSecret(secrets)
mon, err := util.GetMonValFromSecret(secrets)
if err == nil && len(mon) > 0 {
opts.Monitors = mon
}

88
pkg/rbd/controllerserver.go
View File

@ -110,9 +110,15 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
if err := cs.validateVolumeReq(req); err != nil {
return nil, err
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
volumeNameMutex.LockKey(req.GetName())
defer func() {
if err := volumeNameMutex.UnlockKey(req.GetName()); err != nil {
if err = volumeNameMutex.UnlockKey(req.GetName()); err != nil {
klog.Warningf("failed to unlock mutex volume:%s %v", req.GetName(), err)
}
}()
@ -122,7 +128,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
return nil, err
}
found, err := checkVolExists(rbdVol, req.GetSecrets())
found, err := checkVolExists(rbdVol, cr)
if err != nil {
if _, ok := err.(ErrVolNameConflict); ok {
return nil, status.Error(codes.AlreadyExists, err.Error())
@ -140,13 +146,13 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
}, nil
}
err = reserveVol(rbdVol, req.GetSecrets())
err = reserveVol(rbdVol, cr)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
defer func() {
if err != nil {
errDefer := undoVolReservation(rbdVol, req.GetSecrets())
errDefer := undoVolReservation(rbdVol, cr)
if errDefer != nil {
klog.Warningf("failed undoing reservation of volume: %s (%s)", req.GetName(), errDefer)
}
@ -176,7 +182,12 @@ func (cs *ControllerServer) createBackingImage(rbdVol *rbdVolume, req *csi.Creat
return err
}
} else {
err = createImage(rbdVol, volSizeMiB, rbdVol.AdminID, req.GetSecrets())
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return status.Error(codes.Internal, err.Error())
}
err = createImage(rbdVol, volSizeMiB, cr)
if err != nil {
klog.Warningf("failed to create volume: %v", err)
return status.Error(codes.Internal, err.Error())
@ -198,15 +209,20 @@ func (cs *ControllerServer) checkSnapshot(req *csi.CreateVolumeRequest, rbdVol *
return status.Error(codes.InvalidArgument, "volume Snapshot ID cannot be empty")
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return status.Error(codes.Internal, err.Error())
}
rbdSnap := &rbdSnapshot{}
if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil {
if err = genSnapFromSnapID(rbdSnap, snapshotID, cr); err != nil {
if _, ok := err.(ErrSnapNotFound); !ok {
return status.Error(codes.Internal, err.Error())
}
return status.Error(codes.InvalidArgument, "missing requested Snapshot ID")
}
err := restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminID, req.GetSecrets())
err = restoreSnapshot(rbdVol, rbdSnap, cr)
if err != nil {
return status.Error(codes.Internal, err.Error())
}
@ -221,6 +237,12 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
klog.Warningf("invalid delete volume req: %v", protosanitizer.StripSecrets(req))
return nil, err
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
// For now the image get unconditionally deleted, but here retention policy can be checked
volumeID := req.GetVolumeId()
if volumeID == "" {
@ -234,7 +256,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
}()
rbdVol := &rbdVolume{}
if err := genVolFromVolID(rbdVol, volumeID, req.GetSecrets()); err != nil {
if err := genVolFromVolID(rbdVol, volumeID, cr); err != nil {
// if error is ErrKeyNotFound, then a previous attempt at deletion was complete
// or partially complete (image and imageOMap are garbage collected already), hence return
// success as deletion is complete
@ -257,7 +279,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
}
}()
if err := undoVolReservation(rbdVol, req.GetSecrets()); err != nil {
if err := undoVolReservation(rbdVol, cr); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.DeleteVolumeResponse{}, nil
@ -274,7 +296,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
// Deleting rbd image
klog.V(4).Infof("deleting image %s", rbdVol.RbdImageName)
if err := deleteImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil {
if err := deleteImage(rbdVol, cr); err != nil {
klog.Errorf("failed to delete rbd image: %s/%s with error: %v",
rbdVol.Pool, rbdVol.RbdImageName, err)
return nil, status.Error(codes.Internal, err.Error())
@ -314,16 +336,21 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
return nil, err
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
snapshotNameMutex.LockKey(req.GetName())
defer func() {
if err := snapshotNameMutex.UnlockKey(req.GetName()); err != nil {
if err = snapshotNameMutex.UnlockKey(req.GetName()); err != nil {
klog.Warningf("failed to unlock mutex snapshot:%s %v", req.GetName(), err)
}
}()
// Fetch source volume information
rbdVol := new(rbdVolume)
err := genVolFromVolID(rbdVol, req.GetSourceVolumeId(), req.GetSecrets())
err = genVolFromVolID(rbdVol, req.GetSourceVolumeId(), cr)
if err != nil {
if _, ok := err.(ErrImageNotFound); ok {
return nil, status.Errorf(codes.NotFound, "source Volume ID %s not found", req.GetSourceVolumeId())
@ -345,7 +372,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
// Need to check for already existing snapshot name, and if found
// check for the requested source volume id and already allocated source volume id
found, err := checkSnapExists(rbdSnap, req.GetSecrets())
found, err := checkSnapExists(rbdSnap, cr)
if err != nil {
if _, ok := err.(util.ErrSnapNameConflict); ok {
return nil, status.Error(codes.AlreadyExists, err.Error())
@ -365,20 +392,20 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS
}, nil
}
err = reserveSnap(rbdSnap, req.GetSecrets())
err = reserveSnap(rbdSnap, cr)
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
defer func() {
if err != nil {
errDefer := undoSnapReservation(rbdSnap, req.GetSecrets())
errDefer := undoSnapReservation(rbdSnap, cr)
if errDefer != nil {
klog.Warningf("failed undoing reservation of snapshot: %s %v", req.GetName(), errDefer)
}
}
}()
err = cs.doSnapshot(rbdSnap, req.GetSecrets())
err = cs.doSnapshot(rbdSnap, cr)
if err != nil {
return nil, err
}
@ -411,8 +438,8 @@ func (cs *ControllerServer) validateSnapshotReq(req *csi.CreateSnapshotRequest)
return nil
}
func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]string) (err error) {
err = createSnapshot(rbdSnap, rbdSnap.AdminID, secret)
func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, cr *util.Credentials) (err error) {
err = createSnapshot(rbdSnap, cr)
// If snap creation fails, even due to snapname already used, fail, next attempt will get a new
// uuid for use as the snap name
if err != nil {
@ -421,7 +448,7 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s
}
defer func() {
if err != nil {
errDefer := deleteSnapshot(rbdSnap, rbdSnap.AdminID, secret)
errDefer := deleteSnapshot(rbdSnap, cr)
if errDefer != nil {
klog.Errorf("failed to delete snapshot: %v", errDefer)
err = fmt.Errorf("snapshot created but failed to delete snapshot due to"+
@ -431,14 +458,14 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s
}
}()
err = protectSnapshot(rbdSnap, rbdSnap.AdminID, secret)
err = protectSnapshot(rbdSnap, cr)
if err != nil {
klog.Errorf("failed to protect snapshot: %v", err)
return status.Error(codes.Internal, err.Error())
}
defer func() {
if err != nil {
errDefer := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, secret)
errDefer := unprotectSnapshot(rbdSnap, cr)
if errDefer != nil {
klog.Errorf("failed to unprotect snapshot: %v", errDefer)
err = fmt.Errorf("snapshot created but failed to unprotect snapshot due to"+
@ -448,7 +475,7 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s
}
}()
err = getSnapshotMetadata(rbdSnap, rbdSnap.AdminID, secret)
err = getSnapshotMetadata(rbdSnap, cr)
if err != nil {
klog.Errorf("failed to fetch snapshot metadata: %v", err)
return status.Error(codes.Internal, err.Error())
@ -465,6 +492,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
return nil, err
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
snapshotID := req.GetSnapshotId()
if snapshotID == "" {
return nil, status.Error(codes.InvalidArgument, "snapshot ID cannot be empty")
@ -472,18 +504,18 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
snapshotIDMutex.LockKey(snapshotID)
defer func() {
if err := snapshotIDMutex.UnlockKey(snapshotID); err != nil {
if err = snapshotIDMutex.UnlockKey(snapshotID); err != nil {
klog.Warningf("failed to unlock mutex snapshot:%s %v", snapshotID, err)
}
}()
rbdSnap := &rbdSnapshot{}
if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil {
if err = genSnapFromSnapID(rbdSnap, snapshotID, cr); err != nil {
// Consider missing snap as already deleted, and proceed to remove the omap values
if _, ok := err.(ErrSnapNotFound); !ok {
return nil, status.Error(codes.Internal, err.Error())
}
if err := undoSnapReservation(rbdSnap, req.GetSecrets()); err != nil {
if err = undoSnapReservation(rbdSnap, cr); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.DeleteSnapshotResponse{}, nil
@ -493,13 +525,13 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
// cleanup the image and associated omaps for the same
snapshotNameMutex.LockKey(rbdSnap.RequestName)
defer func() {
if err := snapshotNameMutex.UnlockKey(rbdSnap.RequestName); err != nil {
if err = snapshotNameMutex.UnlockKey(rbdSnap.RequestName); err != nil {
klog.Warningf("failed to unlock mutex snapshot:%s %v", rbdSnap.RequestName, err)
}
}()
// Unprotect snapshot
err := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets())
err = unprotectSnapshot(rbdSnap, cr)
if err != nil {
return nil, status.Errorf(codes.FailedPrecondition,
"failed to unprotect snapshot: %s/%s with error: %v",
@ -508,7 +540,7 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS
// Deleting snapshot
klog.V(4).Infof("deleting Snaphot %s", rbdSnap.RbdSnapName)
if err := deleteSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets()); err != nil {
if err := deleteSnapshot(rbdSnap, cr); err != nil {
return nil, status.Errorf(codes.FailedPrecondition,
"failed to delete snapshot: %s/%s with error: %v",
rbdSnap.Pool, rbdSnap.RbdSnapName, err)

9
pkg/rbd/nodeserver.go
View File

@ -57,9 +57,14 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
return nil, status.Error(codes.InvalidArgument, "empty volume ID in request")
}
cr, err := util.GetUserCredentials(req.GetSecrets())
if err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
targetPathMutex.LockKey(targetPath)
defer func() {
if err := targetPathMutex.UnlockKey(targetPath); err != nil {
if err = targetPathMutex.UnlockKey(targetPath); err != nil {
klog.Warningf("failed to unlock mutex targetpath:%s %v", targetPath, err)
}
}()
@ -97,7 +102,7 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
}
volOptions.RbdImageName = volName
// Mapping RBD image
devicePath, err := attachRBDImage(volOptions, volOptions.UserID, req.GetSecrets())
devicePath, err := attachRBDImage(volOptions, cr)
if err != nil {
return nil, err
}

2
pkg/rbd/rbd.go
View File

@ -31,8 +31,6 @@ import (
const (
// volIDVersion is the version number of volume ID encoding scheme
volIDVersion uint16 = 1
rbdDefaultAdminID = "admin"
rbdDefaultUserID = rbdDefaultAdminID
// csiConfigFile is the location of the CSI config file
csiConfigFile = "/etc/ceph-csi-config/config.json"

20
pkg/rbd/rbd_attach.go
View File

@ -25,6 +25,8 @@ import (
"strings"
"time"
"github.com/ceph/ceph-csi/pkg/util"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog"
)
@ -224,7 +226,7 @@ func checkRbdNbdTools() bool {
return true
}
func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string]string) (string, error) {
func attachRBDImage(volOptions *rbdVolume, cr *util.Credentials) (string, error) {
var err error
image := volOptions.RbdImageName
@ -259,26 +261,22 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string
Steps: rbdImageWatcherSteps,
}
err = waitForrbdImage(backoff, volOptions, userID, credentials)
err = waitForrbdImage(backoff, volOptions, cr)
if err != nil {
return "", err
}
devicePath, err = createPath(volOptions, userID, credentials)
devicePath, err = createPath(volOptions, cr)
}
return devicePath, err
}
func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (string, error) {
func createPath(volOpt *rbdVolume, cr *util.Credentials) (string, error) {
image := volOpt.RbdImageName
imagePath := fmt.Sprintf("%s/%s", volOpt.Pool, image)
klog.V(5).Infof("rbd: map mon %s", volOpt.Monitors)
key, err := getKey(userID, creds)
if err != nil {
return "", err
}
useNBD := false
cmdName := rbd
@ -288,7 +286,7 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri
}
output, err := execCommand(cmdName, []string{
"map", imagePath, "--id", userID, "-m", volOpt.Monitors, "--key=" + key})
"map", imagePath, "--id", cr.ID, "-m", volOpt.Monitors, "--key=" + cr.Key})
if err != nil {
klog.Warningf("rbd: map error %v, rbd output: %s", err, string(output))
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
@ -300,12 +298,12 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri
return devicePath, nil
}
func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, credentials map[string]string) error {
func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, cr *util.Credentials) error {
image := volOptions.RbdImageName
imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image)
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
used, rbdOutput, err := rbdStatus(volOptions, userID, credentials)
used, rbdOutput, err := rbdStatus(volOptions, cr)
if err != nil {
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
}

78
pkg/rbd/rbd_journal.go
View File

@ -44,10 +44,6 @@ func validateRbdSnap(rbdSnap *rbdSnapshot) error {
return err
}
if err = validateNonEmptyField(rbdSnap.AdminID, "AdminID", "rbdSnapshot"); err != nil {
return err
}
if err = validateNonEmptyField(rbdSnap.Pool, "Pool", "rbdSnapshot"); err != nil {
return err
}
@ -74,10 +70,6 @@ func validateRbdVol(rbdVol *rbdVolume) error {
return err
}
if err = validateNonEmptyField(rbdVol.AdminID, "AdminID", "rbdVolume"); err != nil {
return err
}
if err = validateNonEmptyField(rbdVol.Pool, "Pool", "rbdVolume"); err != nil {
return err
}
@ -115,18 +107,13 @@ because, the order of omap creation and deletion are inverse of each other, and
request name lock, and hence any stale omaps are leftovers from incomplete transactions and are
hence safe to garbage collect.
*/
func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool, error) {
func checkSnapExists(rbdSnap *rbdSnapshot, cr *util.Credentials) (bool, error) {
err := validateRbdSnap(rbdSnap)
if err != nil {
return false, err
}
key, err := getKey(rbdSnap.AdminID, credentials)
if err != nil {
return false, err
}
snapUUID, err := snapJournal.CheckReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
snapUUID, err := snapJournal.CheckReservation(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.RequestName, rbdSnap.RbdImageName)
if err != nil {
return false, err
@ -137,10 +124,10 @@ func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool,
rbdSnap.RbdSnapName = snapJournal.NamingPrefix() + snapUUID
// Fetch on-disk image attributes
err = updateSnapWithImageInfo(rbdSnap, credentials)
err = updateSnapWithImageInfo(rbdSnap, cr)
if err != nil {
if _, ok := err.(ErrSnapNotFound); ok {
err = snapJournal.UndoReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
err = snapJournal.UndoReservation(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.RbdSnapName, rbdSnap.RequestName)
return false, err
}
@ -148,7 +135,7 @@ func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool,
}
// found a snapshot already available, process and return its information
rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.ClusterID, snapUUID, volIDVersion)
if err != nil {
return false, err
@ -168,18 +155,13 @@ volume names as requested by the CSI drivers. Hence, these need to be invoked on
respective CSI snapshot or volume name based locks are held, as otherwise racy access to these
omaps may end up leaving the omaps in an inconsistent state.
*/
func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, error) {
func checkVolExists(rbdVol *rbdVolume, cr *util.Credentials) (bool, error) {
err := validateRbdVol(rbdVol)
if err != nil {
return false, err
}
key, err := getKey(rbdVol.AdminID, credentials)
if err != nil {
return false, err
}
imageUUID, err := volJournal.CheckReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
imageUUID, err := volJournal.CheckReservation(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.RequestName, "")
if err != nil {
return false, err
@ -193,10 +175,10 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err
// save it for size checks before fetching image data
requestSize := rbdVol.VolSize
// Fetch on-disk image attributes and compare against request
err = updateVolWithImageInfo(rbdVol, credentials)
err = updateVolWithImageInfo(rbdVol, cr)
if err != nil {
if _, ok := err.(ErrImageNotFound); ok {
err = volJournal.UndoReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
err = volJournal.UndoReservation(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.RbdImageName, rbdVol.RequestName)
return false, err
}
@ -212,7 +194,7 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err
// TODO: We should also ensure image features and format is the same
// found a volume already available, process and return it!
rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.ClusterID, imageUUID, volIDVersion)
if err != nil {
return false, err
@ -226,19 +208,14 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err
// reserveSnap is a helper routine to request a rbdSnapshot name reservation and generate the
// volume ID for the generated name
func reserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error {
key, err := getKey(rbdSnap.AdminID, credentials)
if err != nil {
return err
}
snapUUID, err := snapJournal.ReserveName(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
func reserveSnap(rbdSnap *rbdSnapshot, cr *util.Credentials) error {
snapUUID, err := snapJournal.ReserveName(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.RequestName, rbdSnap.RbdImageName)
if err != nil {
return err
}
rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.ClusterID, snapUUID, volIDVersion)
if err != nil {
return err
@ -254,19 +231,14 @@ func reserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error {
// reserveVol is a helper routine to request a rbdVolume name reservation and generate the
// volume ID for the generated name
func reserveVol(rbdVol *rbdVolume, credentials map[string]string) error {
key, err := getKey(rbdVol.AdminID, credentials)
if err != nil {
return err
}
imageUUID, err := volJournal.ReserveName(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
func reserveVol(rbdVol *rbdVolume, cr *util.Credentials) error {
imageUUID, err := volJournal.ReserveName(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.RequestName, "")
if err != nil {
return err
}
rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.ClusterID, imageUUID, volIDVersion)
if err != nil {
return err
@ -281,26 +253,16 @@ func reserveVol(rbdVol *rbdVolume, credentials map[string]string) error {
}
// undoSnapReservation is a helper routine to undo a name reservation for rbdSnapshot
func undoSnapReservation(rbdSnap *rbdSnapshot, credentials map[string]string) error {
key, err := getKey(rbdSnap.AdminID, credentials)
if err != nil {
return err
}
err = snapJournal.UndoReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool,
func undoSnapReservation(rbdSnap *rbdSnapshot, cr *util.Credentials) error {
err := snapJournal.UndoReservation(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.RbdSnapName, rbdSnap.RequestName)
return err
}
// undoVolReservation is a helper routine to undo a name reservation for rbdVolume
func undoVolReservation(rbdVol *rbdVolume, credentials map[string]string) error {
key, err := getKey(rbdVol.AdminID, credentials)
if err != nil {
return err
}
err = volJournal.UndoReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool,
func undoVolReservation(rbdVol *rbdVolume, cr *util.Credentials) error {
err := volJournal.UndoReservation(rbdVol.Monitors, cr, rbdVol.Pool,
rbdVol.RbdImageName, rbdVol.RequestName)
return err

215
pkg/rbd/rbd_util.go
View File

@ -56,8 +56,6 @@ type rbdVolume struct {
ImageFormat string
ImageFeatures string
VolSize int64
AdminID string
UserID string
Mounter string
DisableInUseChecks bool
ClusterID string
@ -79,8 +77,6 @@ type rbdSnapshot struct {
Pool string
CreatedAt *timestamp.Timestamp
SizeBytes int64
AdminID string
UserID string
ClusterID string
RequestName string
}
@ -102,40 +98,23 @@ var (
supportedFeatures = sets.NewString("layering")
)
func getKey(id string, credentials map[string]string) (string, error) {
var (
key string
ok bool
)
if key, ok = credentials[id]; !ok {
return "", fmt.Errorf("RBD key for ID: %s not found", id)
}
return key, nil
}
// createImage creates a new ceph image with provision and volume options.
func createImage(pOpts *rbdVolume, volSz int64, adminID string, credentials map[string]string) error {
func createImage(pOpts *rbdVolume, volSz int64, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
volSzMiB := fmt.Sprintf("%dM", volSz)
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
if pOpts.ImageFormat == rbdImageFormat2 {
klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s ", image, volSzMiB, pOpts.ImageFormat, pOpts.ImageFeatures, pOpts.Monitors, pOpts.Pool)
} else {
klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s", image, volSzMiB, pOpts.ImageFormat, pOpts.Monitors, pOpts.Pool)
}
args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", adminID, "-m", pOpts.Monitors, "--key=" + key, "--image-format", pOpts.ImageFormat}
args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key, "--image-format", pOpts.ImageFormat}
if pOpts.ImageFormat == rbdImageFormat2 {
args = append(args, "--image-feature", pOpts.ImageFeatures)
}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to create rbd image, command output: %s", string(output))
@ -146,21 +125,15 @@ func createImage(pOpts *rbdVolume, volSz int64, adminID string, credentials map[
// rbdStatus checks if there is watcher on the image.
// It returns true if there is a watcher on the image, otherwise returns false.
func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) (bool, string, error) {
func rbdStatus(pOpts *rbdVolume, cr *util.Credentials) (bool, string, error) {
var output string
var cmd []byte
image := pOpts.RbdImageName
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
key, err := getKey(userID, credentials)
if err != nil {
return false, "", err
}
klog.V(4).Infof("rbd: status %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool)
args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userID, "--key=" + key}
cmd, err = execCommand("rbd", args)
args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", cr.ID, "--key=" + cr.Key}
cmd, err := execCommand("rbd", args)
output = string(cmd)
if err, ok := err.(*exec.Error); ok {
@ -185,11 +158,11 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) (
}
// deleteImage deletes a ceph image with provision and volume options.
func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string) error {
func deleteImage(pOpts *rbdVolume, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
found, _, err := rbdStatus(pOpts, adminID, credentials)
found, _, err := rbdStatus(pOpts, cr)
if err != nil {
return err
}
@ -197,21 +170,17 @@ func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string
klog.Info("rbd is still being used ", image)
return fmt.Errorf("rbd %s is still being used", image)
}
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: rm %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool)
args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminID, "-m", pOpts.Monitors,
"--key=" + key}
args := []string{"rm", image, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors,
"--key=" + cr.Key}
output, err = execCommand("rbd", args)
if err != nil {
klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
return err
}
err = undoVolReservation(pOpts, credentials)
err = undoVolReservation(pOpts, cr)
if err != nil {
klog.Errorf("failed to remove reservation for volume (%s) with backing image (%s) (%s)",
pOpts.RequestName, pOpts.RbdImageName, err)
@ -223,14 +192,9 @@ func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string
// updateSnapWithImageInfo updates provided rbdSnapshot with information from on-disk data
// regarding the same
func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, credentials map[string]string) error {
key, err := getKey(rbdSnap.AdminID, credentials)
if err != nil {
return err
}
snapInfo, err := getSnapInfo(rbdSnap.Monitors, rbdSnap.AdminID, key,
rbdSnap.Pool, rbdSnap.RbdImageName, rbdSnap.RbdSnapName)
func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, cr *util.Credentials) error {
snapInfo, err := getSnapInfo(rbdSnap.Monitors, cr, rbdSnap.Pool,
rbdSnap.RbdImageName, rbdSnap.RbdSnapName)
if err != nil {
return err
}
@ -249,14 +213,8 @@ func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, credentials map[string]string
// updateVolWithImageInfo updates provided rbdVolume with information from on-disk data
// regarding the same
func updateVolWithImageInfo(rbdVol *rbdVolume, credentials map[string]string) error {
key, err := getKey(rbdVol.AdminID, credentials)
if err != nil {
return err
}
imageInfo, err := getImageInfo(rbdVol.Monitors, rbdVol.AdminID, key,
rbdVol.Pool, rbdVol.RbdImageName)
func updateVolWithImageInfo(rbdVol *rbdVolume, cr *util.Credentials) error {
imageInfo, err := getImageInfo(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RbdImageName)
if err != nil {
return err
}
@ -275,7 +233,7 @@ func updateVolWithImageInfo(rbdVol *rbdVolume, credentials map[string]string) er
// genSnapFromSnapID generates a rbdSnapshot structure from the provided identifier, updating
// the structure with elements from on-disk snapshot metadata as well
func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, credentials map[string]string) error {
func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, cr *util.Credentials) error {
var (
options map[string]string
vi util.CSIIdentifier
@ -299,32 +257,25 @@ func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, credentials map[
return err
}
rbdSnap.AdminID, rbdSnap.UserID = getIDs(options)
key, err := getKey(rbdSnap.AdminID, credentials)
if err != nil {
return err
}
rbdSnap.Pool, err = util.GetPoolName(rbdSnap.Monitors, rbdSnap.AdminID, key, vi.LocationID)
rbdSnap.Pool, err = util.GetPoolName(rbdSnap.Monitors, cr, vi.LocationID)
if err != nil {
return err
}
rbdSnap.RequestName, rbdSnap.RbdImageName, err = snapJournal.GetObjectUUIDData(rbdSnap.Monitors,
rbdSnap.AdminID, key, rbdSnap.Pool, vi.ObjectUUID, true)
cr, rbdSnap.Pool, vi.ObjectUUID, true)
if err != nil {
return err
}
err = updateSnapWithImageInfo(rbdSnap, credentials)
err = updateSnapWithImageInfo(rbdSnap, cr)
return err
}
// genVolFromVolID generates a rbdVolume structure from the provided identifier, updating
// the structure with elements from on-disk image metadata as well
func genVolFromVolID(rbdVol *rbdVolume, volumeID string, credentials map[string]string) error {
func genVolFromVolID(rbdVol *rbdVolume, volumeID string, cr *util.Credentials) error {
var (
options map[string]string
vi util.CSIIdentifier
@ -351,26 +302,18 @@ func genVolFromVolID(rbdVol *rbdVolume, volumeID string, credentials map[string]
return err
}
rbdVol.AdminID, rbdVol.UserID = getIDs(options)
key, err := getKey(rbdVol.AdminID, credentials)
rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, cr, vi.LocationID)
if err != nil {
return err
}
rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, rbdVol.AdminID, key,
vi.LocationID)
rbdVol.RequestName, _, err = volJournal.GetObjectUUIDData(rbdVol.Monitors, cr,
rbdVol.Pool, vi.ObjectUUID, false)
if err != nil {
return err
}
rbdVol.RequestName, _, err = volJournal.GetObjectUUIDData(rbdVol.Monitors,
rbdVol.AdminID, key, rbdVol.Pool, vi.ObjectUUID, false)
if err != nil {
return err
}
err = updateVolWithImageInfo(rbdVol, credentials)
err = updateVolWithImageInfo(rbdVol, cr)
return err
}
@ -398,26 +341,6 @@ func getMonsAndClusterID(options map[string]string) (monitors, clusterID string,
return
}
func getIDs(options map[string]string) (adminID, userID string) {
var ok bool
adminID, ok = options["adminid"]
switch {
case ok:
default:
adminID = rbdDefaultAdminID
}
userID, ok = options["userid"]
switch {
case ok:
default:
userID = rbdDefaultUserID
}
return adminID, userID
}
func genVolFromVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) {
var (
ok bool
@ -460,20 +383,12 @@ func genVolFromVolumeOptions(volOptions map[string]string, disableInUseChecks bo
klog.V(3).Infof("setting disableInUseChecks on rbd volume to: %v", disableInUseChecks)
rbdVol.DisableInUseChecks = disableInUseChecks
getCredsFromVol(rbdVol, volOptions)
return rbdVol, nil
}
func getCredsFromVol(rbdVol *rbdVolume, volOptions map[string]string) {
var ok bool
rbdVol.AdminID, rbdVol.UserID = getIDs(volOptions)
rbdVol.Mounter, ok = volOptions["mounter"]
if !ok {
rbdVol.Mounter = rbdDefaultMounter
}
return rbdVol, nil
}
func genSnapFromOptions(rbdVol *rbdVolume, snapOptions map[string]string) *rbdSnapshot {
@ -488,8 +403,6 @@ func genSnapFromOptions(rbdVol *rbdVolume, snapOptions map[string]string) *rbdSn
rbdSnap.ClusterID = rbdVol.ClusterID
}
rbdSnap.AdminID, rbdSnap.UserID = getIDs(snapOptions)
return rbdSnap
}
@ -503,22 +416,17 @@ func hasSnapshotFeature(imageFeatures string) bool {
return false
}
func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func protectSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
snapName := pOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s ", image, pOpts.Monitors, pOpts.Pool)
args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
adminID, "-m", pOpts.Monitors, "--key=" + key}
cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to protect snapshot, command output: %s", string(output))
@ -527,21 +435,17 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]
return nil
}
func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func createSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
snapName := pOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: snap create %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool)
args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapName, image,
"--id", adminID, "-m", pOpts.Monitors, "--key=" + key}
"--id", cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to create snapshot, command output: %s", string(output))
@ -550,21 +454,17 @@ func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s
return nil
}
func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func unprotectSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
snapName := pOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool)
args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
adminID, "-m", pOpts.Monitors, "--key=" + key}
cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to unprotect snapshot, command output: %s", string(output))
@ -573,27 +473,23 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[strin
return nil
}
func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func deleteSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error {
var output []byte
image := pOpts.RbdImageName
snapName := pOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool)
args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapName, image, "--id",
adminID, "-m", pOpts.Monitors, "--key=" + key}
cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to delete snapshot, command output: %s", string(output))
}
if err := undoSnapReservation(pOpts, credentials); err != nil {
if err := undoSnapReservation(pOpts, cr); err != nil {
klog.Errorf("failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)",
pOpts.RequestName, pOpts.RbdSnapName, pOpts.RbdImageName, err)
}
@ -601,21 +497,17 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s
return nil
}
func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, cr *util.Credentials) error {
var output []byte
image := pVolOpts.RbdImageName
snapName := pSnapOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
klog.V(4).Infof("rbd: clone %s using mon %s, pool %s", image, pVolOpts.Monitors, pVolOpts.Pool)
args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.RbdImageName + "@" + snapName,
pVolOpts.Pool + "/" + image, "--id", adminID, "-m", pVolOpts.Monitors, "--key=" + key}
pVolOpts.Pool + "/" + image, "--id", cr.ID, "-m", pVolOpts.Monitors, "--key=" + cr.Key}
output, err = execCommand("rbd", args)
output, err := execCommand("rbd", args)
if err != nil {
return errors.Wrapf(err, "failed to restore snapshot, command output: %s", string(output))
@ -626,16 +518,11 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string
// getSnapshotMetadata fetches on-disk metadata about the snapshot and populates the passed in
// rbdSnapshot structure
func getSnapshotMetadata(pSnapOpts *rbdSnapshot, adminID string, credentials map[string]string) error {
func getSnapshotMetadata(pSnapOpts *rbdSnapshot, cr *util.Credentials) error {
imageName := pSnapOpts.RbdImageName
snapName := pSnapOpts.RbdSnapName
key, err := getKey(adminID, credentials)
if err != nil {
return err
}
snapInfo, err := getSnapInfo(pSnapOpts.Monitors, adminID, key, pSnapOpts.Pool, imageName, snapName)
snapInfo, err := getSnapInfo(pSnapOpts.Monitors, cr, pSnapOpts.Pool, imageName, snapName)
if err != nil {
return err
}
@ -666,7 +553,7 @@ type imageInfo struct {
// getImageInfo queries rbd about the given image and returns its metadata, and returns
// ErrImageNotFound if provided image is not found
func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo, error) {
func getImageInfo(monitors string, cr *util.Credentials, poolName, imageName string) (imageInfo, error) {
// rbd --format=json info [image-spec | snap-spec]
var imgInfo imageInfo
@ -674,8 +561,8 @@ func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo
stdout, _, err := util.ExecCommand(
"rbd",
"-m", monitors,
"--id", adminID,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", util.CephConfigPath,
"--format="+"json",
"info", poolName+"/"+imageName)
@ -712,7 +599,7 @@ getSnapInfo queries rbd about the snapshots of the given image and returns its m
returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if provided snap
is not found in the images snapshot list
*/
func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (snapInfo, error) {
func getSnapInfo(monitors string, cr *util.Credentials, poolName, imageName, snapName string) (snapInfo, error) {
// rbd --format=json snap ls [image-spec]
var (
@ -723,8 +610,8 @@ func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (
stdout, _, err := util.ExecCommand(
"rbd",
"-m", monitors,
"--id", adminID,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", util.CephConfigPath,
"--format="+"json",
"snap", "ls", poolName+"/"+imageName)

42
pkg/util/cephcmds.go
View File

@ -55,15 +55,15 @@ type cephStoragePoolSummary struct {
// GetPoolID searches a list of pools in a cluster and returns the ID of the pool that matches
// the passed in poolName parameter
func GetPoolID(monitors, adminID, key, poolName string) (int64, error) {
func GetPoolID(monitors string, cr *Credentials, poolName string) (int64, error) {
// ceph <options> -f json osd lspools
// JSON out: [{"poolnum":<int64>,"poolname":<string>}]
stdout, _, err := ExecCommand(
"ceph",
"-m", monitors,
"--id", adminID,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", CephConfigPath,
"-f", "json",
"osd", "lspools")
@ -90,15 +90,15 @@ func GetPoolID(monitors, adminID, key, poolName string) (int64, error) {
// GetPoolName lists all pools in a ceph cluster, and matches the pool whose pool ID is equal to
// the requested poolID parameter
func GetPoolName(monitors, adminID, key string, poolID int64) (string, error) {
func GetPoolName(monitors string, cr *Credentials, poolID int64) (string, error) {
// ceph <options> -f json osd lspools
// [{"poolnum":1,"poolname":"replicapool"}]
stdout, _, err := ExecCommand(
"ceph",
"-m", monitors,
"--id", adminID,
"--key="+key,
"--id", cr.ID,
"--key="+cr.Key,
"-c", CephConfigPath,
"-f", "json",
"osd", "lspools")
@ -124,12 +124,12 @@ func GetPoolName(monitors, adminID, key string, poolID int64) (string, error) {
}
// SetOMapKeyValue sets the given key and value into the provided Ceph omap name
func SetOMapKeyValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey, keyValue string) error {
func SetOMapKeyValue(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey, keyValue string) error {
// Command: "rados <options> setomapval oMapName oMapKey keyValue"
args := []string{
"-m", monitors,
"--id", adminID,
"--key=" + key,
"--id", cr.ID,
"--key=" + cr.Key,
"-c", CephConfigPath,
"-p", poolName,
"setomapval", oMapName, oMapKey, keyValue,
@ -150,7 +150,7 @@ func SetOMapKeyValue(monitors, adminID, key, poolName, namespace, oMapName, oMap
}
// GetOMapValue gets the value for the given key from the named omap
func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) (string, error) {
func GetOMapValue(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) (string, error) {
// Command: "rados <options> getomapval oMapName oMapKey <outfile>"
// No such key: replicapool/csi.volumes.directory.default/csi.volname
tmpFile, err := ioutil.TempFile("", "omap-get-")
@ -163,8 +163,8 @@ func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey
args := []string{
"-m", monitors,
"--id", adminID,
"--key=" + key,
"--id", cr.ID,
"--key=" + cr.Key,
"-c", CephConfigPath,
"-p", poolName,
"getomapval", oMapName, oMapKey, tmpFile.Name(),
@ -201,12 +201,12 @@ func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey
}
// RemoveOMapKey removes the omap key from the given omap name
func RemoveOMapKey(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) error {
func RemoveOMapKey(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) error {
// Command: "rados <options> rmomapkey oMapName oMapKey"
args := []string{
"-m", monitors,
"--id", adminID,
"--key=" + key,
"--id", cr.ID,
"--key=" + cr.Key,
"-c", CephConfigPath,
"-p", poolName,
"rmomapkey", oMapName, oMapKey,
@ -229,12 +229,12 @@ func RemoveOMapKey(monitors, adminID, key, poolName, namespace, oMapName, oMapKe
// CreateObject creates the object name passed in and returns ErrObjectExists if the provided object
// is already present in rados
func CreateObject(monitors, adminID, key, poolName, namespace, objectName string) error {
func CreateObject(monitors string, cr *Credentials, poolName, namespace, objectName string) error {
// Command: "rados <options> create objectName"
args := []string{
"-m", monitors,
"--id", adminID,
"--key=" + key,
"--id", cr.ID,
"--key=" + cr.Key,
"-c", CephConfigPath,
"-p", poolName,
"create", objectName,
@ -259,12 +259,12 @@ func CreateObject(monitors, adminID, key, poolName, namespace, objectName string
// RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap
// is not found in rados
func RemoveObject(monitors, adminID, key, poolName, namespace, oMapName string) error {
func RemoveObject(monitors string, cr *Credentials, poolName, namespace, oMapName string) error {
// Command: "rados <options> rm oMapName"
args := []string{
"-m", monitors,
"--id", adminID,
"--key=" + key,
"--id", cr.ID,
"--key=" + cr.Key,
"-c", CephConfigPath,
"-p", poolName,
"rm", oMapName,

26
pkg/cephfs/credentials.go → pkg/util/credentials.go
View File

@ -14,9 +14,11 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package cephfs
package util
import "fmt"
import (
"fmt"
)
const (
credUserID = "userID"
@ -26,37 +28,37 @@ const (
credMonitors = "monitors"
)
type credentials struct {
id string
key string
type Credentials struct {
ID string
Key string
}
func getCredentials(idField, keyField string, secrets map[string]string) (*credentials, error) {
func getCredentials(idField, keyField string, secrets map[string]string) (*Credentials, error) {
var (
c = &credentials{}
c = &Credentials{}
ok bool
)
if c.id, ok = secrets[idField]; !ok {
if c.ID, ok = secrets[idField]; !ok {
return nil, fmt.Errorf("missing ID field '%s' in secrets", idField)
}
if c.key, ok = secrets[keyField]; !ok {
if c.Key, ok = secrets[keyField]; !ok {
return nil, fmt.Errorf("missing key field '%s' in secrets", keyField)
}
return c, nil
}
func getUserCredentials(secrets map[string]string) (*credentials, error) {
func GetUserCredentials(secrets map[string]string) (*Credentials, error) {
return getCredentials(credUserID, credUserKey, secrets)
}
func getAdminCredentials(secrets map[string]string) (*credentials, error) {
func GetAdminCredentials(secrets map[string]string) (*Credentials, error) {
return getCredentials(credAdminID, credAdminKey, secrets)
}
func getMonValFromSecret(secrets map[string]string) (string, error) {
func GetMonValFromSecret(secrets map[string]string) (string, error) {
if mons, ok := secrets[credMonitors]; ok {
return mons, nil
}

4
pkg/util/util.go
View File

@ -94,8 +94,8 @@ func ValidateDriverName(driverName string) error {
// GenerateVolID generates a volume ID based on passed in parameters and version, to be returned
// to the CO system
func GenerateVolID(monitors, id, key, pool, clusterID, objUUID string, volIDVersion uint16) (string, error) {
poolID, err := GetPoolID(monitors, id, key, pool)
func GenerateVolID(monitors string, cr *Credentials, pool, clusterID, objUUID string, volIDVersion uint16) (string, error) {
poolID, err := GetPoolID(monitors, cr, pool)
if err != nil {
return "", err
}

36
pkg/util/voljournal.go
View File

@ -175,7 +175,7 @@ Return values:
there was no reservation found
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentName string) (string, error) {
func (cj *CSIJournal) CheckReservation(monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) {
var snapSource bool
if parentName != "" {
@ -187,7 +187,7 @@ func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentN
}
// check if request name is already part of the directory omap
objUUID, err := GetOMapValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
objUUID, err := GetOMapValue(monitors, cr, pool, cj.namespace, cj.csiDirectory,
cj.csiNameKeyPrefix+reqName)
if err != nil {
// error should specifically be not found, for volume to be absent, any other error
@ -198,13 +198,13 @@ func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentN
return "", err
}
savedReqName, savedReqParentName, err := cj.GetObjectUUIDData(monitors, id, key, pool,
savedReqName, savedReqParentName, err := cj.GetObjectUUIDData(monitors, cr, pool,
objUUID, snapSource)
if err != nil {
// error should specifically be not found, for image to be absent, any other error
// is not conclusive, and we should not proceed
if _, ok := err.(ErrKeyNotFound); ok {
err = cj.UndoReservation(monitors, id, key, pool, cj.namingPrefix+objUUID, reqName)
err = cj.UndoReservation(monitors, cr, pool, cj.namingPrefix+objUUID, reqName)
}
return "", err
}
@ -243,11 +243,11 @@ prior to cleaning up the reservation
NOTE: As the function manipulates omaps, it should be called with a lock against the request name
held, to prevent parallel operations from modifying the state of the omaps for this request name.
*/
func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName string) error {
func (cj *CSIJournal) UndoReservation(monitors string, cr *Credentials, pool, volName, reqName string) error {
// delete volume UUID omap (first, inverse of create order)
// TODO: Check cases where volName can be empty, and we need to just cleanup the reqName
imageUUID := strings.TrimPrefix(volName, cj.namingPrefix)
err := RemoveObject(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID)
err := RemoveObject(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID)
if err != nil {
if _, ok := err.(ErrObjectNotFound); !ok {
klog.Errorf("failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+imageUUID, err)
@ -256,7 +256,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName
}
// delete the request name key (last, inverse of create order)
err = RemoveOMapKey(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
err = RemoveOMapKey(monitors, cr, pool, cj.namespace, cj.csiDirectory,
cj.csiNameKeyPrefix+reqName)
if err != nil {
klog.Errorf("failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err)
@ -269,7 +269,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName
// reserveOMapName creates an omap with passed in oMapNamePrefix and a generated <uuid>.
// It ensures generated omap name does not already exist and if conflicts are detected, a set
// number of retires with newer uuids are attempted before returning an error
func reserveOMapName(monitors, id, key, pool, namespace, oMapNamePrefix string) (string, error) {
func reserveOMapName(monitors string, cr *Credentials, pool, namespace, oMapNamePrefix string) (string, error) {
var iterUUID string
maxAttempts := 5
@ -278,7 +278,7 @@ func reserveOMapName(monitors, id, key, pool, namespace, oMapNamePrefix string)
// generate a uuid for the image name
iterUUID = uuid.NewUUID().String()
err := CreateObject(monitors, id, key, pool, namespace, oMapNamePrefix+iterUUID)
err := CreateObject(monitors, cr, pool, namespace, oMapNamePrefix+iterUUID)
if err != nil {
if _, ok := err.(ErrObjectExists); ok {
attempt++
@ -310,7 +310,7 @@ Return values:
- string: Contains the UUID that was reserved for the passed in reqName
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName string) (string, error) {
func (cj *CSIJournal) ReserveName(monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) {
var snapSource bool
if parentName != "" {
@ -325,14 +325,14 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
// NOTE: If any service loss occurs post creation of the UUID directory, and before
// setting the request name key (csiNameKey) to point back to the UUID directory, the
// UUID directory key will be leaked
volUUID, err := reserveOMapName(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix)
volUUID, err := reserveOMapName(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix)
if err != nil {
return "", err
}
// Create request name (csiNameKey) key in csiDirectory and store the UUId based
// volume name into it
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory,
err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.csiDirectory,
cj.csiNameKeyPrefix+reqName, volUUID)
if err != nil {
return "", err
@ -340,7 +340,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
defer func() {
if err != nil {
klog.Warningf("reservation failed for volume: %s", reqName)
errDefer := cj.UndoReservation(monitors, id, key, pool, cj.namingPrefix+volUUID,
errDefer := cj.UndoReservation(monitors, cr, pool, cj.namingPrefix+volUUID,
reqName)
if errDefer != nil {
klog.Warningf("failed undoing reservation of volume: %s (%v)", reqName, errDefer)
@ -349,7 +349,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
}()
// Update UUID directory to store CSI request name
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
cj.csiNameKey, reqName)
if err != nil {
return "", err
@ -357,7 +357,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s
if snapSource {
// Update UUID directory to store source volume UUID in case of snapshots
err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID,
cj.cephSnapSourceKey, parentName)
if err != nil {
return "", err
@ -374,7 +374,7 @@ Return values:
- string: Contains the parent image name for the passed in UUID, if it is a snapshot
- error: non-nil in case of any errors
*/
func (cj *CSIJournal) GetObjectUUIDData(monitors, id, key, pool, objectUUID string, snapSource bool) (string, string, error) {
func (cj *CSIJournal) GetObjectUUIDData(monitors string, cr *Credentials, pool, objectUUID string, snapSource bool) (string, string, error) {
var sourceName string
if snapSource && cj.cephSnapSourceKey == "" {
@ -383,14 +383,14 @@ func (cj *CSIJournal) GetObjectUUIDData(monitors, id, key, pool, objectUUID stri
}
// TODO: fetch all omap vals in one call, than make multiple listomapvals
requestName, err := GetOMapValue(monitors, id, key, pool, cj.namespace,
requestName, err := GetOMapValue(monitors, cr, pool, cj.namespace,
cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiNameKey)
if err != nil {
return "", "", err
}
if snapSource {
sourceName, err = GetOMapValue(monitors, id, key, pool, cj.namespace,
sourceName, err = GetOMapValue(monitors, cr, pool, cj.namespace,
cj.cephUUIDDirectoryPrefix+objectUUID, cj.cephSnapSourceKey)
if err != nil {
return "", "", err

2
scripts/minikube.sh
View File

@ -33,7 +33,7 @@ function install_minikube() {
if type minikube >/dev/null 2>&1; then
local version
version=$(minikube version)
read -ar version <<<"${version}"
read -ra version <<<"${version}"
version=${version[2]}
if [[ "${version}" != "${MINIKUBE_VERSION}" ]]; then
echo "installed minikube version ${version} is not matching requested version ${MINIKUBE_VERSION}"